From 90b77030d5c7da0e88454b503de783bb7e85a080 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Wed, 7 Mar 2018 02:55:47 +0100 Subject: [PATCH 01/23] MyAvatar.hasScriptedBlendshapes property --- interface/src/avatar/MyAvatar.h | 4 ++++ interface/src/avatar/MyHead.cpp | 7 ++++--- libraries/avatars/src/AvatarData.cpp | 3 ++- libraries/avatars/src/AvatarData.h | 2 ++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index fa5206e128..f2d1baf0aa 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -133,6 +133,7 @@ class MyAvatar : public Avatar { Q_PROPERTY(AudioListenerMode audioListenerModeHead READ getAudioListenerModeHead) Q_PROPERTY(AudioListenerMode audioListenerModeCamera READ getAudioListenerModeCamera) Q_PROPERTY(AudioListenerMode audioListenerModeCustom READ getAudioListenerModeCustom) + Q_PROPERTY(bool hasScriptedBlendshapes READ getHasScriptedBlendshapes WRITE setHasScriptedBlendshapes) //TODO: make gravity feature work Q_PROPERTY(glm::vec3 gravity READ getGravity WRITE setGravity) Q_PROPERTY(glm::vec3 leftHandPosition READ getLeftHandPosition) @@ -654,6 +655,8 @@ private: virtual bool shouldRenderHead(const RenderArgs* renderArgs) const override; void setShouldRenderLocally(bool shouldRender) { _shouldRender = shouldRender; setEnableMeshVisible(shouldRender); } bool getShouldRenderLocally() const { return _shouldRender; } + void setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { _hasScriptedBlendShapes = hasScriptedBlendshapes; } + bool getHasScriptedBlendshapes() const override { return _hasScriptedBlendShapes; } bool isMyAvatar() const override { return true; } virtual int parseDataFromBuffer(const QByteArray& buffer) override; virtual glm::vec3 getSkeletonPosition() const override; @@ -764,6 +767,7 @@ private: bool _hmdRollControlEnabled { true }; float _hmdRollControlDeadZone { ROLL_CONTROL_DEAD_ZONE_DEFAULT }; float _hmdRollControlRate { ROLL_CONTROL_RATE_DEFAULT }; + bool _hasScriptedBlendShapes { false }; // working copy -- see AvatarData for thread-safe _sensorToWorldMatrixCache, used for outward facing access glm::mat4 _sensorToWorldMatrix { glm::mat4() }; diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index cad2f9e5d0..b7b8aa5e52 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -46,9 +46,10 @@ void MyHead::simulate(float deltaTime) { auto player = DependencyManager::get(); // Only use face trackers when not playing back a recording. if (!player->isPlaying()) { - FaceTracker* faceTracker = qApp->getActiveFaceTracker(); - _isFaceTrackerConnected = faceTracker != nullptr && !faceTracker->isMuted(); - if (_isFaceTrackerConnected) { + auto faceTracker = qApp->getActiveFaceTracker(); + bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); + _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); + if (hasActualFaceTrackerConnected) { _transientBlendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); if (typeid(*faceTracker) == typeid(DdeFaceTracker)) { diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 1bbc8cc1a5..dcaae59a38 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -300,7 +300,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent tranlationChangedSince(lastSentTime) || parentInfoChangedSince(lastSentTime)); - hasFaceTrackerInfo = !dropFaceTracking && hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); + hasFaceTrackerInfo = !dropFaceTracking && (hasFaceTracker() || getHasScriptedBlendshapes()) && + (sendAll || faceTrackerInfoChangedSince(lastSentTime)); hasJointData = sendAll || !sendMinimum; hasJointDefaultPoseFlags = hasJointData; } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index e927120b07..1f6d2e3630 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -497,6 +497,8 @@ public: float getDomainLimitedScale() const; + virtual bool getHasScriptedBlendshapes() const { return false; } + /**jsdoc * returns the minimum scale allowed for this avatar in the current domain. * This value can change as the user changes avatars or when changing domains. From b0187ecfd3dfdceb084884a72cc1a432f6018ebf Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Tue, 24 Apr 2018 19:24:23 +0200 Subject: [PATCH 02/23] add MyAvatar.hasAudioEnabledFaceMovement API property which is enabled by default --- interface/src/avatar/MyAvatar.h | 4 +++ interface/src/avatar/MyHead.cpp | 45 ++++++++++++++++-------------- libraries/avatars/src/AvatarData.h | 1 + 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index b8c9e4d595..537b5de8d8 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -134,6 +134,7 @@ class MyAvatar : public Avatar { Q_PROPERTY(AudioListenerMode audioListenerModeCamera READ getAudioListenerModeCamera) Q_PROPERTY(AudioListenerMode audioListenerModeCustom READ getAudioListenerModeCustom) Q_PROPERTY(bool hasScriptedBlendshapes READ getHasScriptedBlendshapes WRITE setHasScriptedBlendshapes) + Q_PROPERTY(bool hasAudioEnabledFaceMovement READ getHasAudioEnabledFaceMovement WRITE setHasAudioEnabledFaceMovement) //TODO: make gravity feature work Q_PROPERTY(glm::vec3 gravity READ getGravity WRITE setGravity) Q_PROPERTY(glm::vec3 leftHandPosition READ getLeftHandPosition) @@ -684,6 +685,8 @@ private: bool getShouldRenderLocally() const { return _shouldRender; } void setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { _hasScriptedBlendShapes = hasScriptedBlendshapes; } bool getHasScriptedBlendshapes() const override { return _hasScriptedBlendShapes; } + void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { _hasAudioEnabledFaceMovement = hasAudioEnabledFaceMovement; } + bool getHasAudioEnabledFaceMovement() const override { return _hasAudioEnabledFaceMovement; } bool isMyAvatar() const override { return true; } virtual int parseDataFromBuffer(const QByteArray& buffer) override; virtual glm::vec3 getSkeletonPosition() const override; @@ -793,6 +796,7 @@ private: float _hmdRollControlDeadZone { ROLL_CONTROL_DEAD_ZONE_DEFAULT }; float _hmdRollControlRate { ROLL_CONTROL_RATE_DEFAULT }; bool _hasScriptedBlendShapes { false }; + bool _hasAudioEnabledFaceMovement { true }; // working copy -- see AvatarData for thread-safe _sensorToWorldMatrixCache, used for outward facing access glm::mat4 _sensorToWorldMatrix { glm::mat4() }; diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index b7b8aa5e52..ed38fdd004 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -47,29 +47,32 @@ void MyHead::simulate(float deltaTime) { // Only use face trackers when not playing back a recording. if (!player->isPlaying()) { auto faceTracker = qApp->getActiveFaceTracker(); - bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); + const bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); - if (hasActualFaceTrackerConnected) { - _transientBlendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); - - if (typeid(*faceTracker) == typeid(DdeFaceTracker)) { - - if (Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth)) { - calculateMouthShapes(deltaTime); - - const int JAW_OPEN_BLENDSHAPE = 21; - const int MMMM_BLENDSHAPE = 34; - const int FUNNEL_BLENDSHAPE = 40; - const int SMILE_LEFT_BLENDSHAPE = 28; - const int SMILE_RIGHT_BLENDSHAPE = 29; - _transientBlendshapeCoefficients[JAW_OPEN_BLENDSHAPE] += _audioJawOpen; - _transientBlendshapeCoefficients[SMILE_LEFT_BLENDSHAPE] += _mouth4; - _transientBlendshapeCoefficients[SMILE_RIGHT_BLENDSHAPE] += _mouth4; - _transientBlendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2; - _transientBlendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3; - } - applyEyelidOffset(getFinalOrientationInWorldFrame()); + if (_isFaceTrackerConnected) { + if (hasActualFaceTrackerConnected) { + _transientBlendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); + } else { + _transientBlendshapeCoefficients.fill(0, _blendshapeCoefficients.size()); } + + if (_owningAvatar->getHasAudioEnabledFaceMovement() || (faceTracker && (typeid(*faceTracker) == typeid(DdeFaceTracker)) + && Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth))) { + + calculateMouthShapes(deltaTime); + + const int JAW_OPEN_BLENDSHAPE = 21; + const int MMMM_BLENDSHAPE = 34; + const int FUNNEL_BLENDSHAPE = 40; + const int SMILE_LEFT_BLENDSHAPE = 28; + const int SMILE_RIGHT_BLENDSHAPE = 29; + _transientBlendshapeCoefficients[JAW_OPEN_BLENDSHAPE] += _audioJawOpen; + _transientBlendshapeCoefficients[SMILE_LEFT_BLENDSHAPE] += _mouth4; + _transientBlendshapeCoefficients[SMILE_RIGHT_BLENDSHAPE] += _mouth4; + _transientBlendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2; + _transientBlendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3; + } + applyEyelidOffset(getFinalOrientationInWorldFrame()); } auto eyeTracker = DependencyManager::get(); _isEyeTrackerConnected = eyeTracker->isTracking(); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index fbef083dcd..fe4ff364ae 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -503,6 +503,7 @@ public: float getDomainLimitedScale() const; virtual bool getHasScriptedBlendshapes() const { return false; } + virtual bool getHasAudioEnabledFaceMovement() const { return false; } /**jsdoc * returns the minimum scale allowed for this avatar in the current domain. From 192e48a376026b565ce89b39992b6060fc48d1c6 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Wed, 25 Apr 2018 13:47:00 +0200 Subject: [PATCH 03/23] style fix + JSDocs --- interface/src/avatar/MyAvatar.h | 2 ++ interface/src/avatar/MyHead.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 537b5de8d8..9bccc93fbf 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -80,6 +80,8 @@ class MyAvatar : public Avatar { * MyAvatar.audioListenerModeCamera - the listener is relative to the camera. * MyAvatar.audioListenerModeCustom - the listener is at a custom location specified by the MyAvatar.customListenPosition * and MyAvatar.customListenOrientation properties. + * @property hasScriptedBlendshapes {bool} Blendshapes will be transmitted over the network if set to true. + * @property hasAudioEnabledFaceMovement {bool} If set to true, voice audio will move the mouth Blendshapes while MyAvatar.hasScriptedBlendshapes is enabled. * @property customListenPosition {Vec3} If MyAvatar.audioListenerMode == MyAvatar.audioListenerModeHead, then this determines the position * of audio spatialization listener. * @property customListenOrientation {Quat} If MyAvatar.audioListenerMode == MyAvatar.audioListenerModeHead, then this determines the orientation diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index ed38fdd004..44d679f3f8 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -72,7 +72,7 @@ void MyHead::simulate(float deltaTime) { _transientBlendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2; _transientBlendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3; } - applyEyelidOffset(getFinalOrientationInWorldFrame()); + applyEyelidOffset(getFinalOrientationInWorldFrame()); } auto eyeTracker = DependencyManager::get(); _isEyeTrackerConnected = eyeTracker->isTracking(); From 484e20ea55dc6027b06447e6ca56baeabed88f52 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Wed, 25 Apr 2018 19:01:18 +0200 Subject: [PATCH 04/23] JSDOC - change bool type to boolean, to match JS type --- interface/src/avatar/MyAvatar.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 9bccc93fbf..23faff88ee 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -80,8 +80,8 @@ class MyAvatar : public Avatar { * MyAvatar.audioListenerModeCamera - the listener is relative to the camera. * MyAvatar.audioListenerModeCustom - the listener is at a custom location specified by the MyAvatar.customListenPosition * and MyAvatar.customListenOrientation properties. - * @property hasScriptedBlendshapes {bool} Blendshapes will be transmitted over the network if set to true. - * @property hasAudioEnabledFaceMovement {bool} If set to true, voice audio will move the mouth Blendshapes while MyAvatar.hasScriptedBlendshapes is enabled. + * @property hasScriptedBlendshapes {boolean} Blendshapes will be transmitted over the network if set to true. + * @property hasAudioEnabledFaceMovement {boolean} If set to true, voice audio will move the mouth Blendshapes while MyAvatar.hasScriptedBlendshapes is enabled. * @property customListenPosition {Vec3} If MyAvatar.audioListenerMode == MyAvatar.audioListenerModeHead, then this determines the position * of audio spatialization listener. * @property customListenOrientation {Quat} If MyAvatar.audioListenerMode == MyAvatar.audioListenerModeHead, then this determines the orientation From bbd31d590bb5f465fa5f906ac7dbf756abf7af53 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Thu, 26 Apr 2018 03:12:44 +0200 Subject: [PATCH 05/23] defaults --- interface/src/avatar/MyAvatar.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index ef2b0cd042..512273aee6 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -82,8 +82,8 @@ class MyAvatar : public Avatar { * @property {number} audioListenerModeCamera=1 - The audio listening position is at the camera. Read-only. * @property {number} audioListenerModeCustom=2 - The audio listening position is at a the position specified by set by the * customListenPosition and customListenOrientation property values. Read-only. - * @property hasScriptedBlendshapes {boolean} Blendshapes will be transmitted over the network if set to true. - * @property hasAudioEnabledFaceMovement {boolean} If set to true, voice audio will move the mouth Blendshapes while MyAvatar.hasScriptedBlendshapes is enabled. + * @property {boolean} hasScriptedBlendshapes=false - Blendshapes will be transmitted over the network if set to true. + * @property {boolean} hasAudioEnabledFaceMovement=true - If set to true, voice audio will move the mouth Blendshapes while MyAvatar.hasScriptedBlendshapes is enabled. * @property {Vec3} customListenPosition=Vec3.ZERO - The listening position used when the audioListenerMode * property value is audioListenerModeCustom. * @property {Quat} customListenOrientation=Quat.IDENTITY - The listening orientation used when the From 8b4298514e58530d9c5236cef450ad03431a7d0a Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Tue, 15 May 2018 21:39:17 +0200 Subject: [PATCH 06/23] send out one last blendshape update when setting MyAvatar.hasScriptedBlendshapes to false --- interface/src/avatar/MyAvatar.cpp | 13 +++++++++++++ interface/src/avatar/MyAvatar.h | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 6c6f6d4d41..e4d255f4ee 100755 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -2081,6 +2081,19 @@ bool MyAvatar::shouldRenderHead(const RenderArgs* renderArgs) const { return !defaultMode || !firstPerson || !insideHead; } +void MyAvatar::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { + if (hasScriptedBlendshapes == _hasScriptedBlendShapes) { + return; + } + if (!hasScriptedBlendshapes) { + // send a forced avatarData update to make sure the script can send neutal blendshapes on unload + // without having to wait for the update loop, make sure _hasScriptedBlendShapes is still true + // before sending the update, or else it won't send the neutal blendshapes to the receiving clients + sendAvatarDataPacket(true); + } + _hasScriptedBlendShapes = hasScriptedBlendshapes; +} + void MyAvatar::updateOrientation(float deltaTime) { // Smoothly rotate body with arrow keys diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 76966b5ab3..46cdd7b8ae 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -1345,7 +1345,7 @@ private: virtual bool shouldRenderHead(const RenderArgs* renderArgs) const override; void setShouldRenderLocally(bool shouldRender) { _shouldRender = shouldRender; setEnableMeshVisible(shouldRender); } bool getShouldRenderLocally() const { return _shouldRender; } - void setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { _hasScriptedBlendShapes = hasScriptedBlendshapes; } + void setHasScriptedBlendshapes(bool hasScriptedBlendshapes); bool getHasScriptedBlendshapes() const override { return _hasScriptedBlendShapes; } void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { _hasAudioEnabledFaceMovement = hasAudioEnabledFaceMovement; } bool getHasAudioEnabledFaceMovement() const override { return _hasAudioEnabledFaceMovement; } @@ -1457,7 +1457,7 @@ private: bool _hmdRollControlEnabled { true }; float _hmdRollControlDeadZone { ROLL_CONTROL_DEAD_ZONE_DEFAULT }; float _hmdRollControlRate { ROLL_CONTROL_RATE_DEFAULT }; - bool _hasScriptedBlendShapes { false }; + std::atomic _hasScriptedBlendShapes { false }; bool _hasAudioEnabledFaceMovement { true }; // working copy -- see AvatarData for thread-safe _sensorToWorldMatrixCache, used for outward facing access From 3bda5bf6a0609f23067731cae51210a9e05631e7 Mon Sep 17 00:00:00 2001 From: amantley Date: Thu, 31 May 2018 17:58:37 -0700 Subject: [PATCH 07/23] added MyAvatar.hasProceduralBlinkFaceMovement and MyAvatar.hasProceduralEyeFaceMovement properties to MyAvatar.h and cpp and HeadData.h --- interface/src/avatar/MyAvatar.cpp | 12 + interface/src/avatar/MyAvatar.h | 12 +- .../src/avatars-renderer/Head.cpp | 62 +-- libraries/avatars/src/AvatarData.h | 2 + libraries/avatars/src/HeadData.h | 23 ++ .../DefaultStylizedFemale_Clothed.fst | 139 +++++++ scripts/developer/facialExpressions.js | 374 ++++++++++++++++++ 7 files changed, 596 insertions(+), 28 deletions(-) create mode 100644 scripts/developer/DefaultStylizedFemale_Clothed.fst create mode 100644 scripts/developer/facialExpressions.js diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index e231a42f92..0f6fe88ce9 100755 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -2094,6 +2094,18 @@ void MyAvatar::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { _hasScriptedBlendShapes = hasScriptedBlendshapes; } +void MyAvatar::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { + _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); +} + +void MyAvatar::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { + _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); +} + +void MyAvatar::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { + _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); +} + void MyAvatar::updateOrientation(float deltaTime) { // Smoothly rotate body with arrow keys diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 55780b56ac..813dddcc98 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -87,6 +87,8 @@ class MyAvatar : public Avatar { * @property {number} audioListenerModeCustom=2 - The audio listening position is at a the position specified by set by the * customListenPosition and customListenOrientation property values. Read-only. * @property {boolean} hasScriptedBlendshapes=false - Blendshapes will be transmitted over the network if set to true. + * @property {boolean} hasProceduralBlinkFaceMovement=true - procedural blinking will be turned on if set to true. + * @property {boolean} hasProceduralEyeFaceMovement=true - procedural eye movement will be turned on if set to true. * @property {boolean} hasAudioEnabledFaceMovement=true - If set to true, voice audio will move the mouth Blendshapes while MyAvatar.hasScriptedBlendshapes is enabled. * @property {Vec3} customListenPosition=Vec3.ZERO - The listening position used when the audioListenerMode * property value is audioListenerModeCustom. @@ -187,6 +189,8 @@ class MyAvatar : public Avatar { Q_PROPERTY(AudioListenerMode audioListenerModeCamera READ getAudioListenerModeCamera) Q_PROPERTY(AudioListenerMode audioListenerModeCustom READ getAudioListenerModeCustom) Q_PROPERTY(bool hasScriptedBlendshapes READ getHasScriptedBlendshapes WRITE setHasScriptedBlendshapes) + Q_PROPERTY(bool hasProceduralBlinkFaceMovement READ getHasProceduralBlinkFaceMovement WRITE setHasProceduralBlinkFaceMovement) + Q_PROPERTY(bool hasProceduralEyeFaceMovement READ getHasProceduralEyeFaceMovement WRITE setHasProceduralEyeFaceMovement) Q_PROPERTY(bool hasAudioEnabledFaceMovement READ getHasAudioEnabledFaceMovement WRITE setHasAudioEnabledFaceMovement) //TODO: make gravity feature work Q_PROPERTY(glm::vec3 gravity READ getGravity WRITE setGravity) @@ -1347,8 +1351,12 @@ private: bool getShouldRenderLocally() const { return _shouldRender; } void setHasScriptedBlendshapes(bool hasScriptedBlendshapes); bool getHasScriptedBlendshapes() const override { return _hasScriptedBlendShapes; } - void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { _hasAudioEnabledFaceMovement = hasAudioEnabledFaceMovement; } - bool getHasAudioEnabledFaceMovement() const override { return _hasAudioEnabledFaceMovement; } + void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); + bool getHasProceduralBlinkFaceMovement() const override { return _headData->getHasProceduralBlinkFaceMovement(); } + void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); + bool getHasProceduralEyeFaceMovement() const override { return _headData->getHasProceduralEyeFaceMovement(); } + void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); + bool getHasAudioEnabledFaceMovement() const override { return _headData->getHasAudioEnabledFaceMovement(); } bool isMyAvatar() const override { return true; } virtual int parseDataFromBuffer(const QByteArray& buffer) override; virtual glm::vec3 getSkeletonPosition() const override; diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 256b3bf8a6..b0707922ea 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -100,41 +100,51 @@ void Head::simulate(float deltaTime) { } _browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f); + const float BLINK_SPEED = 10.0f; const float BLINK_SPEED_VARIABILITY = 1.0f; const float BLINK_START_VARIABILITY = 0.25f; const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; - if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { - // no blinking when brows are raised; blink less with increasing loudness - const float BASE_BLINK_RATE = 15.0f / 60.0f; - const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; - if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * + if (getHasProceduralBlinkFaceMovement()) { + if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { + // no blinking when brows are raised; blink less with increasing loudness + const float BASE_BLINK_RATE = 15.0f / 60.0f; + const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; + if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { - _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; - _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; - if (randFloat() < 0.5f) { - _leftEyeBlink = BLINK_START_VARIABILITY; - } else { - _rightEyeBlink = BLINK_START_VARIABILITY; + _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + if (randFloat() < 0.5f) { + _leftEyeBlink = BLINK_START_VARIABILITY; + } + else { + _rightEyeBlink = BLINK_START_VARIABILITY; + } + } + } + else { + _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); + _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); + + if (_leftEyeBlink == FULLY_CLOSED) { + _leftEyeBlinkVelocity = -BLINK_SPEED; + + } + else if (_leftEyeBlink == FULLY_OPEN) { + _leftEyeBlinkVelocity = 0.0f; + } + if (_rightEyeBlink == FULLY_CLOSED) { + _rightEyeBlinkVelocity = -BLINK_SPEED; + + } + else if (_rightEyeBlink == FULLY_OPEN) { + _rightEyeBlinkVelocity = 0.0f; } } } else { - _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); - _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); - - if (_leftEyeBlink == FULLY_CLOSED) { - _leftEyeBlinkVelocity = -BLINK_SPEED; - - } else if (_leftEyeBlink == FULLY_OPEN) { - _leftEyeBlinkVelocity = 0.0f; - } - if (_rightEyeBlink == FULLY_CLOSED) { - _rightEyeBlinkVelocity = -BLINK_SPEED; - - } else if (_rightEyeBlink == FULLY_OPEN) { - _rightEyeBlinkVelocity = 0.0f; - } + _rightEyeBlink = FULLY_OPEN; + _leftEyeBlink = FULLY_OPEN; } // use data to update fake Faceshift blendshape coefficients diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 568c48dd62..2d8ee52ea1 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -502,6 +502,8 @@ public: float getDomainLimitedScale() const; virtual bool getHasScriptedBlendshapes() const { return false; } + virtual bool getHasProceduralBlinkFaceMovement() const { return true; } + virtual bool getHasProceduralEyeFaceMovement() const { return true; } virtual bool getHasAudioEnabledFaceMovement() const { return false; } /**jsdoc diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index bcc2cacde5..f8eca0915e 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -69,6 +69,24 @@ public: } bool lookAtPositionChangedSince(quint64 time) { return _lookAtPositionChanged >= time; } + bool getHasProceduralEyeFaceMovement() const { return _hasProceduralEyeFaceMovement; } + + void setHasProceduralEyeFaceMovement(const bool hasProceduralEyeFaceMovement) { + _hasProceduralEyeFaceMovement = hasProceduralEyeFaceMovement; + } + + bool getHasProceduralBlinkFaceMovement() const { return _hasProceduralBlinkFaceMovement; } + + void setHasProceduralBlinkFaceMovement(const bool hasProceduralBlinkFaceMovement) { + _hasProceduralBlinkFaceMovement = hasProceduralBlinkFaceMovement; + } + + bool getHasAudioEnabledFaceMovement() const { return _hasAudioEnabledFaceMovement; } + + void setHasAudioEnabledFaceMovement(const bool hasAudioEnabledFaceMovement) { + _hasAudioEnabledFaceMovement = hasAudioEnabledFaceMovement; + } + friend class AvatarData; QJsonObject toJson() const; @@ -83,6 +101,11 @@ protected: glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; + //std::atomic _hasProceduralBlinkFaceMovement{ true }; + //std::atomic _hasProceduralEyeFaceMovement{ true }; + bool _hasAudioEnabledFaceMovement { true }; + bool _hasProceduralBlinkFaceMovement{ true }; + bool _hasProceduralEyeFaceMovement{ true }; bool _isFaceTrackerConnected { false }; bool _isEyeTrackerConnected { false }; float _leftEyeBlink { 0.0f }; diff --git a/scripts/developer/DefaultStylizedFemale_Clothed.fst b/scripts/developer/DefaultStylizedFemale_Clothed.fst new file mode 100644 index 0000000000..3e46d6a15c --- /dev/null +++ b/scripts/developer/DefaultStylizedFemale_Clothed.fst @@ -0,0 +1,139 @@ +name = DefaultStylizedFemale_Clothed +type = body+head +scale = 1 +filename = DefaultStylizedFemale_Clothed/DefaultStylizedFemale_Clothed.fbx +texdir = DefaultStylizedFemale_Clothed/textures +joint = jointLean = Spine +joint = jointRightHand = RightHand +joint = jointEyeLeft = LeftEye +joint = jointHead = HeadTop_End +joint = jointNeck = Neck +joint = jointRoot = Hips +joint = jointEyeRight = RightEye +joint = jointLeftHand = LeftHand +freeJoint = LeftArm +freeJoint = LeftForeArm +freeJoint = RightArm +freeJoint = RightForeArm +bs = EyeBlink_L = Blink_Left = 1 +bs = Sneer = Squint_Right = 0.5 +bs = Sneer = Squint_Left = 0.5 +bs = Sneer = NoseScrunch_Right = 0.75 +bs = Sneer = NoseScrunch_Left = 0.75 +bs = ChinLowerRaise = Jaw_Up = 1 +bs = EyeSquint_R = Squint_Right = 1 +bs = MouthSmile_R = Smile_Right = 1 +bs = ChinUpperRaise = UpperLipUp_Right = 0.5 +bs = ChinUpperRaise = UpperLipUp_Left = 0.5 +bs = LipsLowerOpen = LowerLipOut = 1 +bs = LipsLowerDown = LowerLipDown_Right = 0.7 +bs = LipsLowerDown = LowerLipDown_Left = 0.7 +bs = BrowsU_L = BrowsUp_Left = 1 +bs = MouthRight = Midmouth_Right = 1 +bs = MouthDimple_R = Smile_Right = 0.25 +bs = LipsPucker = MouthNarrow_Right = 1 +bs = LipsPucker = MouthNarrow_Left = 1 +bs = Puff = CheekPuff_Right = 1 +bs = Puff = CheekPuff_Left = 1 +bs = JawFwd = JawForeward = 1 +bs = BrowsD_L = BrowsDown_Left = 1 +bs = LipsFunnel = TongueUp = 1 +bs = LipsFunnel = MouthWhistle_NarrowAdjust_Right = 0.5 +bs = LipsFunnel = MouthWhistle_NarrowAdjust_Left = 0.5 +bs = LipsFunnel = MouthNarrow_Right = 1 +bs = LipsFunnel = MouthNarrow_Left = 1 +bs = LipsFunnel = Jaw_Down = 0.36 +bs = LipsFunnel = JawForeward = 0.39 +bs = LipsUpperOpen = UpperLipOut = 1 +bs = EyeSquint_L = Squint_Left = 1 +bs = MouthDimple_L = Smile_Left = 0.25 +bs = LipsLowerClose = LowerLipIn = 1 +bs = MouthFrown_R = Frown_Right = 1 +bs = MouthFrown_L = Frown_Left = 1 +bs = BrowsU_R = BrowsUp_Right = 1 +bs = JawOpen = MouthOpen = 0.7 +bs = JawRight = Jaw_Right = 1 +bs = MouthLeft = Midmouth_Left = 1 +bs = BrowsU_C = BrowsUp_Right = 1 +bs = BrowsU_C = BrowsUp_Left = 1 +bs = LipsUpperUp = UpperLipUp_Right = 0.7 +bs = LipsUpperUp = UpperLipUp_Left = 0.7 +bs = EyeBlink_R = Blink_Right = 1 +bs = EyeOpen_R = EyesWide_Right = 1 +bs = LipsUpperClose = UpperLipIn = 1 +bs = MouthSmile_L = Smile_Left = 1 +bs = EyeOpen_L = EyesWide_Left = 1 +bs = JawLeft = JawRotateY_Left = 0.5 +bs = BrowsD_R = BrowsDown_Right = 1 +jointIndex = RightHandThumb4 = 21 +jointIndex = Neck = 62 +jointIndex = LeftHandIndex4 = 57 +jointIndex = Body = 71 +jointIndex = LeftHandIndex1 = 54 +jointIndex = RightHand = 17 +jointIndex = RightHandMiddle1 = 26 +jointIndex = Spine = 11 +jointIndex = RightHandRing2 = 31 +jointIndex = RightArm = 15 +jointIndex = RightHandPinky2 = 35 +jointIndex = LeftToeBase = 9 +jointIndex = RightHandIndex3 = 24 +jointIndex = RightHandRing1 = 30 +jointIndex = RightHandPinky1 = 34 +jointIndex = RightEye = 66 +jointIndex = LeftHandRing4 = 49 +jointIndex = LeftHandRing2 = 47 +jointIndex = RightHandMiddle2 = 27 +jointIndex = Head = 63 +jointIndex = LeftHandMiddle4 = 53 +jointIndex = LeftLeg = 7 +jointIndex = LeftHandPinky2 = 43 +jointIndex = LeftHandThumb1 = 58 +jointIndex = LeftHandPinky4 = 45 +jointIndex = RightHandIndex1 = 22 +jointIndex = Tops = 67 +jointIndex = Hips = 0 +jointIndex = LeftUpLeg = 6 +jointIndex = RightShoulder = 14 +jointIndex = Spine2 = 13 +jointIndex = RightHandRing4 = 33 +jointIndex = RightHandThumb3 = 20 +jointIndex = RightHandIndex4 = 25 +jointIndex = LeftFoot = 8 +jointIndex = LeftHandRing3 = 48 +jointIndex = LeftHand = 41 +jointIndex = LeftForeArm = 40 +jointIndex = LeftToe_End = 10 +jointIndex = Bottoms = 68 +jointIndex = RightFoot = 3 +jointIndex = LeftHandMiddle2 = 51 +jointIndex = LeftHandThumb3 = 60 +jointIndex = RightHandPinky3 = 36 +jointIndex = LeftEye = 65 +jointIndex = LeftHandIndex2 = 55 +jointIndex = RightHandIndex2 = 23 +jointIndex = LeftHandPinky1 = 42 +jointIndex = LeftHandMiddle3 = 52 +jointIndex = RightHandMiddle4 = 29 +jointIndex = LeftHandThumb2 = 59 +jointIndex = Shoes = 69 +jointIndex = RightHandThumb1 = 18 +jointIndex = RightToe_End = 5 +jointIndex = RightHandThumb2 = 19 +jointIndex = RightUpLeg = 1 +jointIndex = RightLeg = 2 +jointIndex = LeftHandMiddle1 = 50 +jointIndex = LeftHandIndex3 = 56 +jointIndex = LeftHandThumb4 = 61 +jointIndex = RightHandRing3 = 32 +jointIndex = Hair = 70 +jointIndex = Spine1 = 12 +jointIndex = LeftHandRing1 = 46 +jointIndex = LeftArm = 39 +jointIndex = LeftShoulder = 38 +jointIndex = RightForeArm = 16 +jointIndex = HeadTop_End = 64 +jointIndex = RightHandPinky4 = 37 +jointIndex = LeftHandPinky3 = 44 +jointIndex = RightToeBase = 4 +jointIndex = RightHandMiddle3 = 28 diff --git a/scripts/developer/facialExpressions.js b/scripts/developer/facialExpressions.js new file mode 100644 index 0000000000..84e3966c4a --- /dev/null +++ b/scripts/developer/facialExpressions.js @@ -0,0 +1,374 @@ +// +// facialExpressions.js +// A script to set different emotions using blend shapes +// +// Author: Elisa Lupin-Jimenez +// Copyright High Fidelity 2018 +// +// Licensed under the Apache 2.0 License +// See accompanying license file or http://apache.org/ +// +// All assets are under CC Attribution Non-Commerical +// http://creativecommons.org/licenses/ +// + +(function() { + + var TABLET_BUTTON_NAME = "EMOTIONS"; + // TODO: ADD HTML LANDING PAGE + + var TRANSITION_TIME_SECONDS = 0.25; + + var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system"); + var icon = "https://hifi-content.s3.amazonaws.com/elisalj/emoji_scripts/icons/emoji-i.svg"; + var activeIcon = "https://hifi-content.s3.amazonaws.com/elisalj/emoji_scripts/icons/emoji-a.svg"; + var isActive = true; + + var controllerMappingName; + var controllerMapping; + + var tabletButton = tablet.addButton({ + icon: icon, + activeIcon: activeIcon, + text: TABLET_BUTTON_NAME, + isActive: true + }); + + var toggle = function() { + isActive = !isActive; + tabletButton.editProperties({isActive: isActive}); + if (isActive) { + Controller.enableMapping(controllerMappingName); + } else { + setEmotion(DEFAULT); + Controller.disableMapping(controllerMappingName); + } + }; + + tabletButton.clicked.connect(toggle); + + var DEFAULT = { + "EyeOpen_L": 0.00, + "EyeOpen_R": 0.00, + "EyeBlink_L": 0.00, + "EyeBlink_R": 0.00, + "EyeSquint_L": 0.00, + "EyeSquint_R": 0.00, + "BrowsD_L": 0.00, + "BrowsD_R": 0.00, + "BrowsU_L": 0.00, + "BrowsU_C": 0.00, + "JawOpen": 0.00, + "JawFwd": 0.00, + "MouthFrown_L": 0.00, + "MouthFrown_R": 0.00, + "MouthSmile_L": 0.00, + "MouthSmile_R": 0.00, + "MouthDimple_L": 0.00, + "MouthDimple_R": 0.00, + "LipsUpperClose": 0.00, + "LipsLowerClose": 0.00, + "LipsLowerOpen": 0.00, + "ChinUpperRaise": 0.00, + "Sneer": 0.00, + "Puff": 0.00 + }; + + var SMILE = { + "EyeOpen_L": 0.00, + "EyeOpen_R": 0.00, + "EyeBlink_L": 0.30, + "EyeBlink_R": 0.30, + "EyeSquint_L": 0.90, + "EyeSquint_R": 0.90, + "BrowsD_L": 1.00, + "BrowsD_R": 1.00, + "BrowsU_L": 0.00, + "BrowsU_C": 0.00, + "JawOpen": 0.00, + "JawFwd": 0.00, + "MouthFrown_L": 0.00, + "MouthFrown_R": 0.00, + "MouthSmile_L": 1.00, + "MouthSmile_R": 1.00, + "MouthDimple_L": 1.00, + "MouthDimple_R": 1.00, + "LipsUpperClose": 0.40, + "LipsLowerClose": 0.30, + "LipsLowerOpen": 0.25, + "ChinUpperRaise": 0.35, + "Sneer": 0.00, + "Puff": 0.00 + }; + + var LAUGH = { + "EyeOpen_L": 0.00, + "EyeOpen_R": 0.00, + "EyeBlink_L": 0.45, + "EyeBlink_R": 0.45, + "EyeSquint_L": 0.75, + "EyeSquint_R": 0.75, + "BrowsD_L": 0.00, + "BrowsD_R": 0.00, + "BrowsU_L": 0.00, + "BrowsU_C": 0.50, + "JawOpen": 0.50, + "JawFwd": 0.00, + "MouthFrown_L": 0.00, + "MouthFrown_R": 0.00, + "MouthSmile_L": 1.00, + "MouthSmile_R": 1.00, + "MouthDimple_L": 1.00, + "MouthDimple_R": 1.00, + "LipsUpperClose": 0.00, + "LipsLowerClose": 0.00, + "LipsLowerOpen": 0.00, + "ChinUpperRaise": 0.30, + "Sneer": 1.00, + "Puff": 0.30 + }; + + var FLIRT = { + "EyeOpen_L": 0.00, + "EyeOpen_R": 0.00, + "EyeBlink_L": 0.50, + "EyeBlink_R": 0.50, + "EyeSquint_L": 0.25, + "EyeSquint_R": 0.25, + "BrowsD_L": 0.00, + "BrowsD_R": 1.00, + "BrowsU_L": 0.55, + "BrowsU_C": 0.00, + "JawOpen": 0.00, + "JawFwd": 0.00, + "MouthFrown_L": 0.00, + "MouthFrown_R": 0.00, + "MouthSmile_L": 0.50, + "MouthSmile_R": 0.00, + "MouthDimple_L": 1.00, + "MouthDimple_R": 1.00, + "LipsUpperClose": 0.00, + "LipsLowerClose": 0.00, + "LipsLowerOpen": 0.00, + "ChinUpperRaise": 0.00, + "Sneer": 0.00, + "Puff": 0.00 + }; + + var SAD = { + "EyeOpen_L": 0.00, + "EyeOpen_R": 0.00, + "EyeBlink_L": 0.30, + "EyeBlink_R": 0.30, + "EyeSquint_L": 0.30, + "EyeSquint_R": 0.30, + "BrowsD_L": 0.00, + "BrowsD_R": 0.00, + "BrowsU_L": 0.00, + "BrowsU_C": 0.50, + "JawOpen": 0.00, + "JawFwd": 0.80, + "MouthFrown_L": 0.80, + "MouthFrown_R": 0.80, + "MouthSmile_L": 0.00, + "MouthSmile_R": 0.00, + "MouthDimple_L": 0.00, + "MouthDimple_R": 0.00, + "LipsUpperClose": 0.00, + "LipsLowerClose": 0.50, + "LipsLowerOpen": 0.00, + "ChinUpperRaise": 0.00, + "Sneer": 0.00, + "Puff": 0.00 + }; + + var ANGRY = { + "EyeOpen_L": 1.00, + "EyeOpen_R": 1.00, + "EyeBlink_L": 0.00, + "EyeBlink_R": 0.00, + "EyeSquint_L": 1.00, + "EyeSquint_R": 1.00, + "BrowsD_L": 1.00, + "BrowsD_R": 1.00, + "BrowsU_L": 0.00, + "BrowsU_C": 0.00, + "JawOpen": 0.00, + "JawFwd": 0.00, + "MouthFrown_L": 0.50, + "MouthFrown_R": 0.50, + "MouthSmile_L": 0.00, + "MouthSmile_R": 0.00, + "MouthDimple_L": 0.00, + "MouthDimple_R": 0.00, + "LipsUpperClose": 0.50, + "LipsLowerClose": 0.50, + "LipsLowerOpen": 0.00, + "ChinUpperRaise": 0.00, + "Sneer": 0.50, + "Puff": 0.00 + }; + + var FEAR = { + "EyeOpen_L": 1.00, + "EyeOpen_R": 1.00, + "EyeBlink_L": 0.00, + "EyeBlink_R": 0.00, + "EyeSquint_L": 0.00, + "EyeSquint_R": 0.00, + "BrowsD_L": 0.00, + "BrowsD_R": 0.00, + "BrowsU_L": 0.00, + "BrowsU_C": 1.00, + "JawOpen": 0.15, + "JawFwd": 0.00, + "MouthFrown_L": 0.30, + "MouthFrown_R": 0.30, + "MouthSmile_L": 0.00, + "MouthSmile_R": 0.00, + "MouthDimple_L": 0.00, + "MouthDimple_R": 0.00, + "LipsUpperClose": 0.00, + "LipsLowerClose": 0.00, + "LipsLowerOpen": 0.00, + "ChinUpperRaise": 0.00, + "Sneer": 0.00, + "Puff": 0.00 + }; + + var DISGUST = { + "EyeOpen_L": 0.00, + "EyeOpen_R": 0.00, + "EyeBlink_L": 0.25, + "EyeBlink_R": 0.25, + "EyeSquint_L": 1.00, + "EyeSquint_R": 1.00, + "BrowsD_L": 1.00, + "BrowsD_R": 1.00, + "BrowsU_L": 0.00, + "BrowsU_C": 0.00, + "JawOpen": 0.00, + "JawFwd": 0.00, + "MouthFrown_L": 1.00, + "MouthFrown_R": 1.00, + "MouthSmile_L": 0.00, + "MouthSmile_R": 0.00, + "MouthDimple_L": 0.00, + "MouthDimple_R": 0.00, + "LipsUpperClose": 0.00, + "LipsLowerClose": 0.75, + "LipsLowerOpen": 0.00, + "ChinUpperRaise": 0.75, + "Sneer": 1.00, + "Puff": 0.00 + }; + + + function mixValue(valueA, valueB, percentage) { + return valueA + ((valueB - valueA) * percentage); + } + + var lastEmotionUsed = DEFAULT; + var emotion = DEFAULT; + var isChangingEmotion = false; + var changingEmotionPercentage = 0.0; + + Script.update.connect(function(deltaTime) { + if (!isChangingEmotion) { + return; + } + changingEmotionPercentage += deltaTime / TRANSITION_TIME_SECONDS; + if (changingEmotionPercentage >= 1.0) { + changingEmotionPercentage = 1.0; + isChangingEmotion = false; + if (emotion === DEFAULT) { + MyAvatar.hasScriptedBlendshapes = false; + } + } + for (var blendshape in emotion) { + MyAvatar.setBlendshape(blendshape, + mixValue(lastEmotionUsed[blendshape], emotion[blendshape], changingEmotionPercentage)); + } + }); + + function setEmotion(currentEmotion) { + if (emotion !== lastEmotionUsed) { + lastEmotionUsed = emotion; + } + if (currentEmotion !== lastEmotionUsed) { + changingEmotionPercentage = 0.0; + emotion = currentEmotion; + isChangingEmotion = true; + MyAvatar.hasScriptedBlendshapes = true; + } + } + + + controllerMappingName = 'Hifi-FacialExpressions-Mapping'; + controllerMapping = Controller.newMapping(controllerMappingName); + + controllerMapping.from(Controller.Hardware.Keyboard.H).to(function(value) { + if (value !== 0) { + setEmotion(SMILE); + } + }); + + controllerMapping.from(Controller.Hardware.Keyboard.J).to(function(value) { + if (value !== 0) { + setEmotion(LAUGH); + } + }); + + controllerMapping.from(Controller.Hardware.Keyboard.K).to(function(value) { + if (value !== 0) { + setEmotion(FLIRT); + } + }); + + controllerMapping.from(Controller.Hardware.Keyboard.L).to(function(value) { + if (value !== 0) { + setEmotion(SAD); + } + }); + + controllerMapping.from(Controller.Hardware.Keyboard.V).to(function(value) { + if (value !== 0) { + setEmotion(ANGRY); + } + }); + + controllerMapping.from(Controller.Hardware.Keyboard.B).to(function(value) { + if (value !== 0) { + setEmotion(FEAR); + } + }); + + controllerMapping.from(Controller.Hardware.Keyboard.M).to(function(value) { + if (value !== 0) { + setEmotion(DISGUST); + } + }); + + controllerMapping.from(Controller.Hardware.Keyboard.N).to(function(value) { + if (value !== 0) { + setEmotion(DEFAULT); + } + }); + + Controller.enableMapping(controllerMappingName); + + Script.scriptEnding.connect(function() { + tabletButton.clicked.disconnect(toggle); + tablet.removeButton(tabletButton); + Controller.disableMapping(controllerMappingName); + + if (emotion !== DEFAULT || isChangingEmotion) { + isChangingEmotion = false; + for (var blendshape in DEFAULT) { + MyAvatar.setBlendshape(blendshape, DEFAULT[blendshape]); + } + MyAvatar.hasScriptedBlendshapes = false; + } + }); + +}()); \ No newline at end of file From 01848fafe0b716872a8dd781c7c7d3f6faacc2ff Mon Sep 17 00:00:00 2001 From: amantley Date: Fri, 1 Jun 2018 11:37:11 -0700 Subject: [PATCH 08/23] made all changes except those to AvatarData toByteArray function also to do: verify that transientBlendshapes are set to 0 in headData simulate when there are blendshapes from script but no face tracker present --- interface/src/avatar/MyHead.cpp | 2 +- .../src/avatars-renderer/Head.cpp | 30 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 44d679f3f8..13d635bdd8 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -51,7 +51,7 @@ void MyHead::simulate(float deltaTime) { _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); if (_isFaceTrackerConnected) { if (hasActualFaceTrackerConnected) { - _transientBlendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); + _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); } else { _transientBlendshapeCoefficients.fill(0, _blendshapeCoefficients.size()); } diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index b0707922ea..06b52c8d44 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -148,18 +148,28 @@ void Head::simulate(float deltaTime) { } // use data to update fake Faceshift blendshape coefficients - calculateMouthShapes(deltaTime); + if (getHasAudioEnabledFaceMovement()) { + calculateMouthShapes(deltaTime); + } else { + _audioJawOpen = 0.0f; + _browAudioLift = 0.0f; + _mouth2 = 0.0f; + _mouth3 = 0.0f; + _mouth4 = 0.0f; + _mouthTime = 0.0f; + } FaceTracker::updateFakeCoefficients(_leftEyeBlink, - _rightEyeBlink, - _browAudioLift, - _audioJawOpen, - _mouth2, - _mouth3, - _mouth4, - _transientBlendshapeCoefficients); - - applyEyelidOffset(getOrientation()); + _rightEyeBlink, + _browAudioLift, + _audioJawOpen, + _mouth2, + _mouth3, + _mouth4, + _transientBlendshapeCoefficients); + if (getHasProceduralEyeFaceMovement()) { + applyEyelidOffset(getOrientation()); + } } else { _saccade = glm::vec3(); } From 2879eeb687d4885cf220799bbade8fcd3960aaf1 Mon Sep 17 00:00:00 2001 From: amantley Date: Fri, 1 Jun 2018 17:18:48 -0700 Subject: [PATCH 09/23] added flags for procedural face movement AvatarData.cpp toByteArray and parseDataFromBuffer --- libraries/avatars/src/AvatarData.cpp | 54 ++++++++++++++++++++-------- libraries/avatars/src/AvatarData.h | 7 ++-- libraries/shared/src/SharedUtil.cpp | 17 ++++++--- libraries/shared/src/SharedUtil.h | 6 ++-- 4 files changed, 61 insertions(+), 23 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 649460d6c7..247fc2a76a 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -308,7 +308,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + - (hasFaceTrackerInfo ? AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getNumSummedBlendshapeCoefficients()) : 0) + + (hasFaceTrackerInfo ? AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) : 0) + (hasJointData ? AvatarDataPacket::maxJointDataSize(_jointData.size()) : 0) + (hasJointDefaultPoseFlags ? AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()) : 0); @@ -443,7 +443,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent auto startSection = destinationBuffer; auto data = reinterpret_cast(destinationBuffer); - uint8_t flags { 0 }; + uint16_t flags { 0 }; setSemiNibbleAt(flags, KEY_STATE_START_BIT, _keyState); @@ -451,20 +451,33 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent bool isFingerPointing = _handState & IS_FINGER_POINTING_FLAG; setSemiNibbleAt(flags, HAND_STATE_START_BIT, _handState & ~IS_FINGER_POINTING_FLAG); if (isFingerPointing) { - setAtBit(flags, HAND_STATE_FINGER_POINTING_BIT); + setAtBit16(flags, HAND_STATE_FINGER_POINTING_BIT); } // face tracker state if (_headData->_isFaceTrackerConnected) { - setAtBit(flags, IS_FACE_TRACKER_CONNECTED); + setAtBit16(flags, IS_FACE_TRACKER_CONNECTED); } // eye tracker state if (_headData->_isEyeTrackerConnected) { - setAtBit(flags, IS_EYE_TRACKER_CONNECTED); + setAtBit16(flags, IS_EYE_TRACKER_CONNECTED); } // referential state if (!parentID.isNull()) { - setAtBit(flags, HAS_REFERENTIAL); + setAtBit16(flags, HAS_REFERENTIAL); } + // audio face movement + if (_headData->getHasAudioEnabledFaceMovement()) { + setAtBit16(flags, AUDIO_ENABLED_FACE_MOVEMENT); + } + // procedural eye face movement + if (_headData->getHasProceduralEyeFaceMovement()) { + setAtBit16(flags, PROCEDURAL_EYE_FACE_MOVEMENT); + } + // procedural blink face movement + if (_headData->getHasProceduralBlinkFaceMovement()) { + setAtBit16(flags, PROCEDURAL_BLINK_FACE_MOVEMENT); + } + data->flags = flags; destinationBuffer += sizeof(AvatarDataPacket::AdditionalFlags); @@ -507,7 +520,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent if (hasFaceTrackerInfo) { auto startSection = destinationBuffer; auto faceTrackerInfo = reinterpret_cast(destinationBuffer); - const auto& blendshapeCoefficients = _headData->getSummedBlendshapeCoefficients(); + const auto& blendshapeCoefficients = _headData->getBlendshapeCoefficients(); faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; @@ -973,7 +986,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { PACKET_READ_CHECK(AdditionalFlags, sizeof(AvatarDataPacket::AdditionalFlags)); auto data = reinterpret_cast(sourceBuffer); - uint8_t bitItems = data->flags; + uint16_t bitItems = data->flags; // key state, stored as a semi-nibble in the bitItems auto newKeyState = (KeyState)getSemiNibbleAt(bitItems, KEY_STATE_START_BIT); @@ -981,26 +994,37 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { // hand state, stored as a semi-nibble plus a bit in the bitItems // we store the hand state as well as other items in a shared bitset. The hand state is an octal, but is split // into two sections to maintain backward compatibility. The bits are ordered as such (0-7 left to right). - // +---+-----+-----+--+ - // |x,x|H0,H1|x,x,x|H2| - // +---+-----+-----+--+ + // AA 6/1/18 added three more flags here for procedural audio, blink, and eye saccade enabled + // +---+-----+-----+--+--+--+--+-----+ + // |x,x|H0,H1|x,x,x|H2|Au|Bl|Ey|xxxxx| + // +---+-----+-----+--+--+--+--+-----+ // Hand state - H0,H1,H2 is found in the 3rd, 4th, and 8th bits auto newHandState = getSemiNibbleAt(bitItems, HAND_STATE_START_BIT) - + (oneAtBit(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0); + + (oneAtBit16(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0); - auto newFaceTrackerConnected = oneAtBit(bitItems, IS_FACE_TRACKER_CONNECTED); - auto newEyeTrackerConnected = oneAtBit(bitItems, IS_EYE_TRACKER_CONNECTED); + auto newFaceTrackerConnected = oneAtBit16(bitItems, IS_FACE_TRACKER_CONNECTED); + auto newEyeTrackerConnected = oneAtBit16(bitItems, IS_EYE_TRACKER_CONNECTED); + + auto newHasAudioEnabledFaceMovement = oneAtBit16(bitItems, AUDIO_ENABLED_FACE_MOVEMENT); + auto newHasProceduralEyeFaceMovement = oneAtBit16(bitItems, PROCEDURAL_EYE_FACE_MOVEMENT); + auto newHasProceduralBlinkFaceMovement = oneAtBit16(bitItems, PROCEDURAL_BLINK_FACE_MOVEMENT); bool keyStateChanged = (_keyState != newKeyState); bool handStateChanged = (_handState != newHandState); bool faceStateChanged = (_headData->_isFaceTrackerConnected != newFaceTrackerConnected); bool eyeStateChanged = (_headData->_isEyeTrackerConnected != newEyeTrackerConnected); - bool somethingChanged = keyStateChanged || handStateChanged || faceStateChanged || eyeStateChanged; + bool audioEnableFaceMovementChanged = (_headData->getHasAudioEnabledFaceMovement() != newHasAudioEnabledFaceMovement); + bool proceduralEyeFaceMovementChanged = (_headData->getHasProceduralEyeFaceMovement() != newHasProceduralEyeFaceMovement); + bool proceduralBlinkFaceMovementChanged = (_headData->getHasProceduralBlinkFaceMovement() != newHasProceduralBlinkFaceMovement); + bool somethingChanged = keyStateChanged || handStateChanged || faceStateChanged || eyeStateChanged || audioEnableFaceMovementChanged || proceduralEyeFaceMovementChanged || proceduralBlinkFaceMovementChanged; _keyState = newKeyState; _handState = newHandState; _headData->_isFaceTrackerConnected = newFaceTrackerConnected; _headData->_isEyeTrackerConnected = newEyeTrackerConnected; + _headData->setHasAudioEnabledFaceMovement(newHasAudioEnabledFaceMovement); + _headData->setHasProceduralEyeFaceMovement(newHasProceduralEyeFaceMovement); + _headData->setHasProceduralBlinkFaceMovement(newHasProceduralBlinkFaceMovement); sourceBuffer += sizeof(AvatarDataPacket::AdditionalFlags); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 2d8ee52ea1..ad304d5796 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -93,6 +93,9 @@ const int IS_FACE_TRACKER_CONNECTED = 4; // 5th bit const int IS_EYE_TRACKER_CONNECTED = 5; // 6th bit (was CHAT_CIRCLING) const int HAS_REFERENTIAL = 6; // 7th bit const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit +const int AUDIO_ENABLED_FACE_MOVEMENT = 8; // 9th bit +const int PROCEDURAL_EYE_FACE_MOVEMENT = 9; // 10th bit +const int PROCEDURAL_BLINK_FACE_MOVEMENT = 10; // 11th bit const char HAND_STATE_NULL = 0; @@ -200,9 +203,9 @@ namespace AvatarDataPacket { static_assert(sizeof(SensorToWorldMatrix) == SENSOR_TO_WORLD_SIZE, "AvatarDataPacket::SensorToWorldMatrix size doesn't match."); PACKED_BEGIN struct AdditionalFlags { - uint8_t flags; // additional flags: hand state, key state, eye tracking + uint16_t flags; // additional flags: hand state, key state, eye tracking } PACKED_END; - const size_t ADDITIONAL_FLAGS_SIZE = 1; + const size_t ADDITIONAL_FLAGS_SIZE = 2; static_assert(sizeof(AdditionalFlags) == ADDITIONAL_FLAGS_SIZE, "AvatarDataPacket::AdditionalFlags size doesn't match."); // only present if HAS_REFERENTIAL flag is set in AvatarInfo.flags diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index 7c4a9b6d6c..35aab3fac1 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -297,14 +297,23 @@ void setAtBit(unsigned char& byte, int bitIndex) { byte |= (1 << (7 - bitIndex)); } +bool oneAtBit16(unsigned short word, int bitIndex) { + return (word >> (7 - bitIndex) & 1); +} + +void setAtBit16(unsigned short& word, int bitIndex) { + word |= (1 << (7 - bitIndex)); +} + + void clearAtBit(unsigned char& byte, int bitIndex) { if (oneAtBit(byte, bitIndex)) { byte -= (1 << (7 - bitIndex)); } } -int getSemiNibbleAt(unsigned char byte, int bitIndex) { - return (byte >> (6 - bitIndex) & 3); // semi-nibbles store 00, 01, 10, or 11 +int getSemiNibbleAt(unsigned short word, int bitIndex) { + return (word >> (6 - bitIndex) & 3); // semi-nibbles store 00, 01, 10, or 11 } int getNthBit(unsigned char byte, int ordinal) { @@ -326,9 +335,9 @@ int getNthBit(unsigned char byte, int ordinal) { return ERROR_RESULT; } -void setSemiNibbleAt(unsigned char& byte, int bitIndex, int value) { +void setSemiNibbleAt(unsigned short& word, int bitIndex, int value) { //assert(value <= 3 && value >= 0); - byte |= ((value & 3) << (6 - bitIndex)); // semi-nibbles store 00, 01, 10, or 11 + word |= ((value & 3) << (6 - bitIndex)); // semi-nibbles store 00, 01, 10, or 11 } bool isInEnvironment(const char* environment) { diff --git a/libraries/shared/src/SharedUtil.h b/libraries/shared/src/SharedUtil.h index 9875314aa4..db9fff3e93 100644 --- a/libraries/shared/src/SharedUtil.h +++ b/libraries/shared/src/SharedUtil.h @@ -163,9 +163,11 @@ void printVoxelCode(unsigned char* voxelCode); int numberOfOnes(unsigned char byte); bool oneAtBit(unsigned char byte, int bitIndex); void setAtBit(unsigned char& byte, int bitIndex); +bool oneAtBit16(unsigned short word, int bitIndex); +void setAtBit16(unsigned short& word, int bitIndex); void clearAtBit(unsigned char& byte, int bitIndex); -int getSemiNibbleAt(unsigned char byte, int bitIndex); -void setSemiNibbleAt(unsigned char& byte, int bitIndex, int value); +int getSemiNibbleAt(unsigned short word, int bitIndex); +void setSemiNibbleAt(unsigned short& word, int bitIndex, int value); int getNthBit(unsigned char byte, int ordinal); /// determines the bit placement 0-7 of the ordinal set bit From beaea7abe6bcf2f76c0ef24962e3df3c12217980 Mon Sep 17 00:00:00 2001 From: Angus Antley Date: Sat, 2 Jun 2018 14:39:30 +0100 Subject: [PATCH 10/23] changed max face tracker info size to reflect summed coeff --- libraries/avatars/src/AvatarData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 247fc2a76a..4a3967a938 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -308,7 +308,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + - (hasFaceTrackerInfo ? AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) : 0) + + (hasFaceTrackerInfo ? AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getNumSummedBlendshapeCoefficients()) : 0) + (hasJointData ? AvatarDataPacket::maxJointDataSize(_jointData.size()) : 0) + (hasJointDefaultPoseFlags ? AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()) : 0); From 61592cdadac1277e129ec378160c63aa80578586 Mon Sep 17 00:00:00 2001 From: amantley Date: Mon, 4 Jun 2018 16:37:47 -0700 Subject: [PATCH 11/23] completed first try at all changes for scripted blendshapes --- interface/src/avatar/MyHead.cpp | 4 +++- libraries/avatars/src/AvatarData.cpp | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 13d635bdd8..8f53522b5a 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -72,7 +72,9 @@ void MyHead::simulate(float deltaTime) { _transientBlendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2; _transientBlendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3; } - applyEyelidOffset(getFinalOrientationInWorldFrame()); + if (_owningAvatar->getHasProceduralEyeFaceMovement()) { + applyEyelidOffset(getFinalOrientationInWorldFrame()); + } } auto eyeTracker = DependencyManager::get(); _isEyeTrackerConnected = eyeTracker->isTracking(); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 247fc2a76a..7ae9ba1257 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1102,6 +1102,8 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { int numBytesRead = sourceBuffer - startSection; _faceTrackerRate.increment(numBytesRead); _faceTrackerUpdateRate.increment(); + } else { + _headData->_blendshapeCoefficients.fill(0, _headData->_blendshapeCoefficients.size()); } if (hasJointData) { From 5132466ca0c4d3f5ac623643f6ee3b707010167b Mon Sep 17 00:00:00 2001 From: amantley Date: Tue, 5 Jun 2018 13:32:29 -0700 Subject: [PATCH 12/23] kept the transient coeff. and sending the blendshapes accross the network. and removed procedural updates to face from myhead.cpp --- interface/src/avatar/MyHead.cpp | 29 +--- .../src/avatars-renderer/Head.cpp | 160 +++++++++--------- 2 files changed, 81 insertions(+), 108 deletions(-) diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 8f53522b5a..55db478d85 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -50,32 +50,11 @@ void MyHead::simulate(float deltaTime) { const bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); if (_isFaceTrackerConnected) { - if (hasActualFaceTrackerConnected) { - _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); - } else { - _transientBlendshapeCoefficients.fill(0, _blendshapeCoefficients.size()); - } - - if (_owningAvatar->getHasAudioEnabledFaceMovement() || (faceTracker && (typeid(*faceTracker) == typeid(DdeFaceTracker)) - && Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth))) { - - calculateMouthShapes(deltaTime); - - const int JAW_OPEN_BLENDSHAPE = 21; - const int MMMM_BLENDSHAPE = 34; - const int FUNNEL_BLENDSHAPE = 40; - const int SMILE_LEFT_BLENDSHAPE = 28; - const int SMILE_RIGHT_BLENDSHAPE = 29; - _transientBlendshapeCoefficients[JAW_OPEN_BLENDSHAPE] += _audioJawOpen; - _transientBlendshapeCoefficients[SMILE_LEFT_BLENDSHAPE] += _mouth4; - _transientBlendshapeCoefficients[SMILE_RIGHT_BLENDSHAPE] += _mouth4; - _transientBlendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2; - _transientBlendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3; - } - if (_owningAvatar->getHasProceduralEyeFaceMovement()) { - applyEyelidOffset(getFinalOrientationInWorldFrame()); - } + _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); + } else { + _blendshapeCoefficients.fill(0, _blendshapeCoefficients.size()); } + auto eyeTracker = DependencyManager::get(); _isEyeTrackerConnected = eyeTracker->isTracking(); } diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 06b52c8d44..81b58c7aa1 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -58,24 +58,32 @@ void Head::simulate(float deltaTime) { _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); } - if (!_isFaceTrackerConnected) { - if (!_isEyeTrackerConnected) { - // Update eye saccades - const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; - const float AVERAGE_SACCADE_INTERVAL = 6.0f; - const float MICROSACCADE_MAGNITUDE = 0.002f; - const float SACCADE_MAGNITUDE = 0.04f; - const float NOMINAL_FRAME_RATE = 60.0f; + //if (!_isFaceTrackerConnected) { + if (!_isEyeTrackerConnected) { + // Update eye saccades + const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; + const float AVERAGE_SACCADE_INTERVAL = 6.0f; + const float MICROSACCADE_MAGNITUDE = 0.002f; + const float SACCADE_MAGNITUDE = 0.04f; + const float NOMINAL_FRAME_RATE = 60.0f; - if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { - _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); - } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { - _saccadeTarget = SACCADE_MAGNITUDE * randVector(); - } - _saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime); - } else { - _saccade = glm::vec3(); + if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { + _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); + } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { + _saccadeTarget = SACCADE_MAGNITUDE * randVector(); } + _saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime); + } else { + _saccade = glm::vec3(); + } + + + const float BLINK_SPEED = 10.0f; + const float BLINK_SPEED_VARIABILITY = 1.0f; + const float BLINK_START_VARIABILITY = 0.25f; + const float FULLY_OPEN = 0.0f; + const float FULLY_CLOSED = 1.0f; + if (getHasProceduralBlinkFaceMovement()) { // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; @@ -88,26 +96,7 @@ void Head::simulate(float deltaTime) { forceBlink = true; } - // Update audio attack data for facial animation (eyebrows and mouth) - float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz - _audioAttack = audioAttackAveragingRate * _audioAttack + - (1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness); - _lastLoudness = (audioLoudness - _longTermAverageLoudness); - - const float BROW_LIFT_THRESHOLD = 100.0f; - if (_audioAttack > BROW_LIFT_THRESHOLD) { - _browAudioLift += sqrtf(_audioAttack) * 0.01f; - } - _browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f); - - - const float BLINK_SPEED = 10.0f; - const float BLINK_SPEED_VARIABILITY = 1.0f; - const float BLINK_START_VARIABILITY = 0.25f; - const float FULLY_OPEN = 0.0f; - const float FULLY_CLOSED = 1.0f; - if (getHasProceduralBlinkFaceMovement()) { - if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { + if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { // no blinking when brows are raised; blink less with increasing loudness const float BASE_BLINK_RATE = 15.0f / 60.0f; const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; @@ -117,63 +106,68 @@ void Head::simulate(float deltaTime) { _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; if (randFloat() < 0.5f) { _leftEyeBlink = BLINK_START_VARIABILITY; - } - else { + } else { _rightEyeBlink = BLINK_START_VARIABILITY; } } - } - else { - _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); - _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); - - if (_leftEyeBlink == FULLY_CLOSED) { - _leftEyeBlinkVelocity = -BLINK_SPEED; - - } - else if (_leftEyeBlink == FULLY_OPEN) { - _leftEyeBlinkVelocity = 0.0f; - } - if (_rightEyeBlink == FULLY_CLOSED) { - _rightEyeBlinkVelocity = -BLINK_SPEED; - - } - else if (_rightEyeBlink == FULLY_OPEN) { - _rightEyeBlinkVelocity = 0.0f; - } - } } else { - _rightEyeBlink = FULLY_OPEN; - _leftEyeBlink = FULLY_OPEN; - } + _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); + _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); - // use data to update fake Faceshift blendshape coefficients - if (getHasAudioEnabledFaceMovement()) { - calculateMouthShapes(deltaTime); - } else { - _audioJawOpen = 0.0f; - _browAudioLift = 0.0f; - _mouth2 = 0.0f; - _mouth3 = 0.0f; - _mouth4 = 0.0f; - _mouthTime = 0.0f; - } - FaceTracker::updateFakeCoefficients(_leftEyeBlink, - _rightEyeBlink, - _browAudioLift, - _audioJawOpen, - _mouth2, - _mouth3, - _mouth4, - _transientBlendshapeCoefficients); + if (_leftEyeBlink == FULLY_CLOSED) { + _leftEyeBlinkVelocity = -BLINK_SPEED; - if (getHasProceduralEyeFaceMovement()) { - applyEyelidOffset(getOrientation()); + } else if (_leftEyeBlink == FULLY_OPEN) { + _leftEyeBlinkVelocity = 0.0f; + } + if (_rightEyeBlink == FULLY_CLOSED) { + _rightEyeBlinkVelocity = -BLINK_SPEED; + + } else if (_rightEyeBlink == FULLY_OPEN) { + _rightEyeBlinkVelocity = 0.0f; + } } } else { - _saccade = glm::vec3(); + _rightEyeBlink = FULLY_OPEN; + _leftEyeBlink = FULLY_OPEN; } + // use data to update fake Faceshift blendshape coefficients + if (getHasAudioEnabledFaceMovement()) { + // Update audio attack data for facial animation (eyebrows and mouth) + float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz + _audioAttack = audioAttackAveragingRate * _audioAttack + + (1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness); + _lastLoudness = (audioLoudness - _longTermAverageLoudness); + const float BROW_LIFT_THRESHOLD = 100.0f; + if (_audioAttack > BROW_LIFT_THRESHOLD) { + _browAudioLift += sqrtf(_audioAttack) * 0.01f; + } + _browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f); + calculateMouthShapes(deltaTime); + + } else { + _audioJawOpen = 0.0f; + _browAudioLift = 0.0f; + _mouth2 = 0.0f; + _mouth3 = 0.0f; + _mouth4 = 0.0f; + _mouthTime = 0.0f; + } + + FaceTracker::updateFakeCoefficients(_leftEyeBlink, + _rightEyeBlink, + _browAudioLift, + _audioJawOpen, + _mouth2, + _mouth3, + _mouth4, + _transientBlendshapeCoefficients); + + if (getHasProceduralEyeFaceMovement()) { + applyEyelidOffset(getOrientation()); + } + _leftEyePosition = _rightEyePosition = getPosition(); if (_owningAvatar) { auto skeletonModel = static_cast(_owningAvatar)->getSkeletonModel(); From f21431ca88c17020f09932e9ee2fb9985029d4b9 Mon Sep 17 00:00:00 2001 From: amantley Date: Tue, 5 Jun 2018 16:33:28 -0700 Subject: [PATCH 13/23] added check for the actual face tracker in MyHead.cpp to prevent crash --- interface/src/avatar/MyHead.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 55db478d85..eb365c20fb 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -50,9 +50,9 @@ void MyHead::simulate(float deltaTime) { const bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); if (_isFaceTrackerConnected) { - _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); - } else { - _blendshapeCoefficients.fill(0, _blendshapeCoefficients.size()); + if (hasActualFaceTrackerConnected) { + _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); + } } auto eyeTracker = DependencyManager::get(); From 0c879d85b5d89b00ff3daf4bbed090e64bd97381 Mon Sep 17 00:00:00 2001 From: amantley Date: Tue, 5 Jun 2018 17:33:32 -0700 Subject: [PATCH 14/23] added print statements and removed summed coeff from avatarData.cpp --- libraries/avatars-renderer/src/avatars-renderer/Head.cpp | 5 ++++- libraries/avatars/src/AvatarData.cpp | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 81b58c7aa1..39b50b9255 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -20,6 +20,7 @@ #include #include #include +#include "Logging.h" #include "Avatar.h" @@ -84,7 +85,7 @@ void Head::simulate(float deltaTime) { const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; if (getHasProceduralBlinkFaceMovement()) { - + qCDebug(avatars_renderer) << "in the blink code"; // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; const float TALKING_LOUDNESS = 100.0f; @@ -134,6 +135,7 @@ void Head::simulate(float deltaTime) { // use data to update fake Faceshift blendshape coefficients if (getHasAudioEnabledFaceMovement()) { + qCDebug(avatars_renderer) << "in the audio face code"; // Update audio attack data for facial animation (eyebrows and mouth) float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz _audioAttack = audioAttackAveragingRate * _audioAttack + @@ -165,6 +167,7 @@ void Head::simulate(float deltaTime) { _transientBlendshapeCoefficients); if (getHasProceduralEyeFaceMovement()) { + qCDebug(avatars_renderer) << "in the eye face code"; applyEyelidOffset(getOrientation()); } diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index d72abe592b..7ae9ba1257 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -308,7 +308,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + - (hasFaceTrackerInfo ? AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getNumSummedBlendshapeCoefficients()) : 0) + + (hasFaceTrackerInfo ? AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) : 0) + (hasJointData ? AvatarDataPacket::maxJointDataSize(_jointData.size()) : 0) + (hasJointDefaultPoseFlags ? AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()) : 0); From dca93ca61f8fe2ef51f6ba0a809a3dd59a7e364a Mon Sep 17 00:00:00 2001 From: amantley Date: Wed, 6 Jun 2018 16:44:12 -0700 Subject: [PATCH 15/23] fixed setatone16 function to return the correct result for additional flags --- .../avatars-renderer/src/avatars-renderer/Head.cpp | 14 +++++++++++--- libraries/avatars/src/AvatarData.cpp | 11 +++++++++++ libraries/shared/src/SharedUtil.cpp | 4 ++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 39b50b9255..9b6333a4b5 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -45,6 +45,13 @@ void Head::reset() { void Head::simulate(float deltaTime) { const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for + qCDebug(avatars_renderer) << "name " << _owningAvatar->getName(); + if (_owningAvatar->isMyAvatar()) { + qCDebug(avatars_renderer) << "my avatar"; + } else { + qCDebug(avatars_renderer) << "not my avatar " << _owningAvatar->getAudioLoudness(); + } + // grab the audio loudness from the owning avatar, if we have one float audioLoudness = _owningAvatar ? _owningAvatar->getAudioLoudness() : 0.0f; @@ -78,6 +85,7 @@ void Head::simulate(float deltaTime) { _saccade = glm::vec3(); } + const float BLINK_SPEED = 10.0f; const float BLINK_SPEED_VARIABILITY = 1.0f; @@ -85,7 +93,7 @@ void Head::simulate(float deltaTime) { const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; if (getHasProceduralBlinkFaceMovement()) { - qCDebug(avatars_renderer) << "in the blink code"; + qCDebug(avatars_renderer) << "in the blink code " << _owningAvatar->getName(); // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; const float TALKING_LOUDNESS = 100.0f; @@ -135,7 +143,7 @@ void Head::simulate(float deltaTime) { // use data to update fake Faceshift blendshape coefficients if (getHasAudioEnabledFaceMovement()) { - qCDebug(avatars_renderer) << "in the audio face code"; + //qCDebug(avatars_renderer) << "in the audio face code"; // Update audio attack data for facial animation (eyebrows and mouth) float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz _audioAttack = audioAttackAveragingRate * _audioAttack + @@ -167,7 +175,7 @@ void Head::simulate(float deltaTime) { _transientBlendshapeCoefficients); if (getHasProceduralEyeFaceMovement()) { - qCDebug(avatars_renderer) << "in the eye face code"; + //qCDebug(avatars_renderer) << "in the eye face code"; applyEyelidOffset(getOrientation()); } diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 7ae9ba1257..43b490719f 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -522,12 +522,17 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent auto faceTrackerInfo = reinterpret_cast(destinationBuffer); const auto& blendshapeCoefficients = _headData->getBlendshapeCoefficients(); + //for (int i = 0; i < blendshapeCoefficients.size(); i++) { + // qCWarning(avatars) << "blend coeff " << i << " " << blendshapeCoefficients[i]; + //} + faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; faceTrackerInfo->averageLoudness = _headData->_averageLoudness; faceTrackerInfo->browAudioLift = _headData->_browAudioLift; faceTrackerInfo->numBlendshapeCoefficients = blendshapeCoefficients.size(); destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); + qCWarning(avatars) << "face tracker info left eye blink " << faceTrackerInfo->leftEyeBlink; memcpy(destinationBuffer, blendshapeCoefficients.data(), blendshapeCoefficients.size() * sizeof(float)); destinationBuffer += blendshapeCoefficients.size() * sizeof(float); @@ -1009,6 +1014,11 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { auto newHasProceduralEyeFaceMovement = oneAtBit16(bitItems, PROCEDURAL_EYE_FACE_MOVEMENT); auto newHasProceduralBlinkFaceMovement = oneAtBit16(bitItems, PROCEDURAL_BLINK_FACE_MOVEMENT); + if (newHasAudioEnabledFaceMovement) { + qCWarning(avatars) << "name " << getName() << "audio enabled flag is true"; + } else { + qCWarning(avatars) << "name " << getName() << "audio enabled flag is false"; + } bool keyStateChanged = (_keyState != newKeyState); bool handStateChanged = (_handState != newHandState); bool faceStateChanged = (_headData->_isFaceTrackerConnected != newFaceTrackerConnected); @@ -1086,6 +1096,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { PACKET_READ_CHECK(FaceTrackerInfo, sizeof(AvatarDataPacket::FaceTrackerInfo)); auto faceTrackerInfo = reinterpret_cast(sourceBuffer); sourceBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); + qCWarning(avatars) << "parse data left eye blink " << faceTrackerInfo->leftEyeBlink; _headData->_leftEyeBlink = faceTrackerInfo->leftEyeBlink; _headData->_rightEyeBlink = faceTrackerInfo->rightEyeBlink; diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index 35aab3fac1..c0203106c0 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -298,11 +298,11 @@ void setAtBit(unsigned char& byte, int bitIndex) { } bool oneAtBit16(unsigned short word, int bitIndex) { - return (word >> (7 - bitIndex) & 1); + return (word >> (16 - bitIndex) & 1); } void setAtBit16(unsigned short& word, int bitIndex) { - word |= (1 << (7 - bitIndex)); + word |= (1 << (16 - bitIndex)); } From 6a11ddc3497b9cd3a393e24ed8ae6bc96f1c97da Mon Sep 17 00:00:00 2001 From: amantley Date: Wed, 6 Jun 2018 17:50:50 -0700 Subject: [PATCH 16/23] removed print statements and removed the setting of procedural head variables in avatarData.cpp --- .../src/avatars-renderer/Head.cpp | 14 ++--- libraries/avatars/src/AvatarData.cpp | 55 +++++++++++++++---- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 9b6333a4b5..538c928902 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -45,12 +45,12 @@ void Head::reset() { void Head::simulate(float deltaTime) { const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for - qCDebug(avatars_renderer) << "name " << _owningAvatar->getName(); - if (_owningAvatar->isMyAvatar()) { - qCDebug(avatars_renderer) << "my avatar"; - } else { - qCDebug(avatars_renderer) << "not my avatar " << _owningAvatar->getAudioLoudness(); - } + //qCDebug(avatars_renderer) << "name " << _owningAvatar->getName(); + //if (_owningAvatar->isMyAvatar()) { + // qCDebug(avatars_renderer) << "my avatar"; + //} else { + // qCDebug(avatars_renderer) << "not my avatar " << _owningAvatar->getAudioLoudness(); + //} // grab the audio loudness from the owning avatar, if we have one float audioLoudness = _owningAvatar ? _owningAvatar->getAudioLoudness() : 0.0f; @@ -93,7 +93,7 @@ void Head::simulate(float deltaTime) { const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; if (getHasProceduralBlinkFaceMovement()) { - qCDebug(avatars_renderer) << "in the blink code " << _owningAvatar->getName(); + //qCDebug(avatars_renderer) << "in the blink code " << _owningAvatar->getName(); // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; const float TALKING_LOUDNESS = 100.0f; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 43b490719f..feb73fd0fb 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -532,7 +532,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent faceTrackerInfo->browAudioLift = _headData->_browAudioLift; faceTrackerInfo->numBlendshapeCoefficients = blendshapeCoefficients.size(); destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); - qCWarning(avatars) << "face tracker info left eye blink " << faceTrackerInfo->leftEyeBlink; + //qCWarning(avatars) << "face tracker info left eye blink " << faceTrackerInfo->leftEyeBlink; memcpy(destinationBuffer, blendshapeCoefficients.data(), blendshapeCoefficients.size() * sizeof(float)); destinationBuffer += blendshapeCoefficients.size() * sizeof(float); @@ -1014,11 +1014,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { auto newHasProceduralEyeFaceMovement = oneAtBit16(bitItems, PROCEDURAL_EYE_FACE_MOVEMENT); auto newHasProceduralBlinkFaceMovement = oneAtBit16(bitItems, PROCEDURAL_BLINK_FACE_MOVEMENT); - if (newHasAudioEnabledFaceMovement) { - qCWarning(avatars) << "name " << getName() << "audio enabled flag is true"; - } else { - qCWarning(avatars) << "name " << getName() << "audio enabled flag is false"; - } + bool keyStateChanged = (_keyState != newKeyState); bool handStateChanged = (_handState != newHandState); bool faceStateChanged = (_headData->_isFaceTrackerConnected != newFaceTrackerConnected); @@ -1040,6 +1036,43 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { if (somethingChanged) { _additionalFlagsChanged = usecTimestampNow(); + if (newHasAudioEnabledFaceMovement) { + qCWarning(avatars) << "name " << getName() << "audio enabled flag is true"; + } else { + qCWarning(avatars) << "name " << getName() << "audio enabled flag is false"; + } + if (newHasProceduralEyeFaceMovement) { + qCWarning(avatars) << "name " << getName() << "eye face enabled flag is true"; + } else { + qCWarning(avatars) << "name " << getName() << "eye face flag is false"; + } + if (newHasProceduralBlinkFaceMovement) { + qCWarning(avatars) << "name " << getName() << "blink flag is true"; + } else { + qCWarning(avatars) << "name " << getName() << "blink flag is false"; + } + if (newFaceTrackerConnected) { + qCWarning(avatars) << "name " << getName() << "face tracker flag is true"; + } else { + qCWarning(avatars) << "name " << getName() << "face tracker flag is false"; + } + if (newEyeTrackerConnected) { + qCWarning(avatars) << "name " << getName() << "eye tracker flag is true"; + } else { + qCWarning(avatars) << "name " << getName() << "eye tracker flag is false"; + } + if (newHandState) { + qCWarning(avatars) << "name " << getName() << "hand state flag is true"; + } else { + qCWarning(avatars) << "name " << getName() << "hand state flag is false"; + } + if (newKeyState) { + qCWarning(avatars) << "name " << getName() << "key state flag is true"; + } else { + qCWarning(avatars) << "name " << getName() << "key state flag is false"; + } + + } int numBytesRead = sourceBuffer - startSection; _additionalFlagsRate.increment(numBytesRead); @@ -1096,12 +1129,12 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { PACKET_READ_CHECK(FaceTrackerInfo, sizeof(AvatarDataPacket::FaceTrackerInfo)); auto faceTrackerInfo = reinterpret_cast(sourceBuffer); sourceBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); - qCWarning(avatars) << "parse data left eye blink " << faceTrackerInfo->leftEyeBlink; + //qCWarning(avatars) << "parse data left eye blink " << faceTrackerInfo->leftEyeBlink; - _headData->_leftEyeBlink = faceTrackerInfo->leftEyeBlink; - _headData->_rightEyeBlink = faceTrackerInfo->rightEyeBlink; - _headData->_averageLoudness = faceTrackerInfo->averageLoudness; - _headData->_browAudioLift = faceTrackerInfo->browAudioLift; + //_headData->_leftEyeBlink = faceTrackerInfo->leftEyeBlink; + //_headData->_rightEyeBlink = faceTrackerInfo->rightEyeBlink; + //_headData->_averageLoudness = faceTrackerInfo->averageLoudness; + //_headData->_browAudioLift = faceTrackerInfo->browAudioLift; int numCoefficients = faceTrackerInfo->numBlendshapeCoefficients; const int coefficientsSize = sizeof(float) * numCoefficients; From 97831e61f054d496d1c9b3740bd8e36ccc4a18f6 Mon Sep 17 00:00:00 2001 From: amantley Date: Thu, 7 Jun 2018 12:54:46 -0700 Subject: [PATCH 17/23] removed stray comments --- interface/src/avatar/MyHead.cpp | 1 + .../src/avatars-renderer/Head.cpp | 3 --- libraries/avatars/src/AvatarData.cpp | 23 ++++++++----------- libraries/avatars/src/AvatarData.h | 13 ++++++++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index eb365c20fb..960dfd3402 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -57,6 +57,7 @@ void MyHead::simulate(float deltaTime) { auto eyeTracker = DependencyManager::get(); _isEyeTrackerConnected = eyeTracker->isTracking(); + // if eye tracker is connected we should get the data here. } Parent::simulate(deltaTime); } diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 538c928902..07d82f8af7 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -93,7 +93,6 @@ void Head::simulate(float deltaTime) { const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; if (getHasProceduralBlinkFaceMovement()) { - //qCDebug(avatars_renderer) << "in the blink code " << _owningAvatar->getName(); // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; const float TALKING_LOUDNESS = 100.0f; @@ -143,7 +142,6 @@ void Head::simulate(float deltaTime) { // use data to update fake Faceshift blendshape coefficients if (getHasAudioEnabledFaceMovement()) { - //qCDebug(avatars_renderer) << "in the audio face code"; // Update audio attack data for facial animation (eyebrows and mouth) float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz _audioAttack = audioAttackAveragingRate * _audioAttack + @@ -175,7 +173,6 @@ void Head::simulate(float deltaTime) { _transientBlendshapeCoefficients); if (getHasProceduralEyeFaceMovement()) { - //qCDebug(avatars_renderer) << "in the eye face code"; applyEyelidOffset(getOrientation()); } diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index feb73fd0fb..539e109501 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -522,17 +522,14 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent auto faceTrackerInfo = reinterpret_cast(destinationBuffer); const auto& blendshapeCoefficients = _headData->getBlendshapeCoefficients(); - //for (int i = 0; i < blendshapeCoefficients.size(); i++) { - // qCWarning(avatars) << "blend coeff " << i << " " << blendshapeCoefficients[i]; - //} - + //note: we don't use the blink and average loudness, we just use the numBlendShapes and + // compute the procedural info on the client side. faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; faceTrackerInfo->averageLoudness = _headData->_averageLoudness; faceTrackerInfo->browAudioLift = _headData->_browAudioLift; faceTrackerInfo->numBlendshapeCoefficients = blendshapeCoefficients.size(); destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); - //qCWarning(avatars) << "face tracker info left eye blink " << faceTrackerInfo->leftEyeBlink; memcpy(destinationBuffer, blendshapeCoefficients.data(), blendshapeCoefficients.size() * sizeof(float)); destinationBuffer += blendshapeCoefficients.size() * sizeof(float); @@ -999,7 +996,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { // hand state, stored as a semi-nibble plus a bit in the bitItems // we store the hand state as well as other items in a shared bitset. The hand state is an octal, but is split // into two sections to maintain backward compatibility. The bits are ordered as such (0-7 left to right). - // AA 6/1/18 added three more flags here for procedural audio, blink, and eye saccade enabled + // AA 6/1/18 added three more flags bits 8,9, and 10 for procedural audio, blink, and eye saccade enabled // +---+-----+-----+--+--+--+--+-----+ // |x,x|H0,H1|x,x,x|H2|Au|Bl|Ey|xxxxx| // +---+-----+-----+--+--+--+--+-----+ @@ -1124,25 +1121,23 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { } if (hasFaceTrackerInfo) { + //qCWarning(avatars) << "parsing face tracker info "; auto startSection = sourceBuffer; PACKET_READ_CHECK(FaceTrackerInfo, sizeof(AvatarDataPacket::FaceTrackerInfo)); auto faceTrackerInfo = reinterpret_cast(sourceBuffer); - sourceBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); - //qCWarning(avatars) << "parse data left eye blink " << faceTrackerInfo->leftEyeBlink; - - //_headData->_leftEyeBlink = faceTrackerInfo->leftEyeBlink; - //_headData->_rightEyeBlink = faceTrackerInfo->rightEyeBlink; - //_headData->_averageLoudness = faceTrackerInfo->averageLoudness; - //_headData->_browAudioLift = faceTrackerInfo->browAudioLift; - int numCoefficients = faceTrackerInfo->numBlendshapeCoefficients; const int coefficientsSize = sizeof(float) * numCoefficients; + sourceBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); + PACKET_READ_CHECK(FaceTrackerCoefficients, coefficientsSize); _headData->_blendshapeCoefficients.resize(numCoefficients); // make sure there's room for the copy! _headData->_transientBlendshapeCoefficients.resize(numCoefficients); + + //only copy the blendshapes to headData not the procedural face info memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, coefficientsSize); sourceBuffer += coefficientsSize; + int numBytesRead = sourceBuffer - startSection; _faceTrackerRate.increment(numBytesRead); _faceTrackerUpdateRate.increment(); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index ad304d5796..51b3257ba2 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -79,14 +79,21 @@ const quint32 AVATAR_MOTION_SCRIPTABLE_BITS = // Bitset of state flags - we store the key state, hand state, Faceshift, eye tracking, and existence of // referential data in this bit set. The hand state is an octal, but is split into two sections to maintain // backward compatibility. The bits are ordered as such (0-7 left to right). -// +-----+-----+-+-+-+--+ -// |K0,K1|H0,H1|F|E|R|H2| -// +-----+-----+-+-+-+--+ +// AA 6/1/18 added three more flags bits 8,9, and 10 for procedural audio, blink, and eye saccade enabled +// +// +-----+-----+-+-+-+--+--+--+--+-----+ +// |K0,K1|H0,H1|F|E|R|H2|Au|Bl|Ey|xxxxx| +// +-----+-----+-+-+-+--+--+--+--+-----+ +// // Key state - K0,K1 is found in the 1st and 2nd bits // Hand state - H0,H1,H2 is found in the 3rd, 4th, and 8th bits // Face tracker - F is found in the 5th bit // Eye tracker - E is found in the 6th bit // Referential Data - R is found in the 7th bit +// Procedural audio to mouth movement is enabled 8th bit +// Procedural Blink is enabled 9th bit +// Procedural Eyelid is enabled 10th bit + const int KEY_STATE_START_BIT = 0; // 1st and 2nd bits const int HAND_STATE_START_BIT = 2; // 3rd and 4th bits const int IS_FACE_TRACKER_CONNECTED = 4; // 5th bit From 836c3da8581c6db43cbda0754cefd2266dff137f Mon Sep 17 00:00:00 2001 From: amantley Date: Thu, 7 Jun 2018 15:11:57 -0700 Subject: [PATCH 18/23] cleaned up white space and removed extra declaration of hadAudioEnabledFaceMovement --- interface/src/avatar/MyAvatar.h | 1 - interface/src/avatar/MyHead.cpp | 2 +- .../src/avatars-renderer/Head.cpp | 17 +-- libraries/avatars/src/AvatarData.cpp | 11 +- libraries/avatars/src/HeadData.h | 2 - .../DefaultStylizedFemale_Clothed.fst | 139 ------------------ scripts/developer/facialExpressions.js | 4 +- 7 files changed, 10 insertions(+), 166 deletions(-) delete mode 100644 scripts/developer/DefaultStylizedFemale_Clothed.fst diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 813dddcc98..b401b2469a 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -1466,7 +1466,6 @@ private: float _hmdRollControlDeadZone { ROLL_CONTROL_DEAD_ZONE_DEFAULT }; float _hmdRollControlRate { ROLL_CONTROL_RATE_DEFAULT }; std::atomic _hasScriptedBlendShapes { false }; - bool _hasAudioEnabledFaceMovement { true }; // working copy -- see AvatarData for thread-safe _sensorToWorldMatrixCache, used for outward facing access glm::mat4 _sensorToWorldMatrix { glm::mat4() }; diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 960dfd3402..9b05a26c76 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -54,7 +54,7 @@ void MyHead::simulate(float deltaTime) { _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); } } - + auto eyeTracker = DependencyManager::get(); _isEyeTrackerConnected = eyeTracker->isTracking(); // if eye tracker is connected we should get the data here. diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 07d82f8af7..50f20918fa 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -45,13 +45,6 @@ void Head::reset() { void Head::simulate(float deltaTime) { const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for - //qCDebug(avatars_renderer) << "name " << _owningAvatar->getName(); - //if (_owningAvatar->isMyAvatar()) { - // qCDebug(avatars_renderer) << "my avatar"; - //} else { - // qCDebug(avatars_renderer) << "not my avatar " << _owningAvatar->getAudioLoudness(); - //} - // grab the audio loudness from the owning avatar, if we have one float audioLoudness = _owningAvatar ? _owningAvatar->getAudioLoudness() : 0.0f; @@ -84,9 +77,7 @@ void Head::simulate(float deltaTime) { } else { _saccade = glm::vec3(); } - - - + const float BLINK_SPEED = 10.0f; const float BLINK_SPEED_VARIABILITY = 1.0f; const float BLINK_START_VARIABILITY = 0.25f; @@ -162,7 +153,7 @@ void Head::simulate(float deltaTime) { _mouth4 = 0.0f; _mouthTime = 0.0f; } - + FaceTracker::updateFakeCoefficients(_leftEyeBlink, _rightEyeBlink, _browAudioLift, @@ -171,11 +162,11 @@ void Head::simulate(float deltaTime) { _mouth3, _mouth4, _transientBlendshapeCoefficients); - + if (getHasProceduralEyeFaceMovement()) { applyEyelidOffset(getOrientation()); } - + _leftEyePosition = _rightEyePosition = getPosition(); if (_owningAvatar) { auto skeletonModel = static_cast(_owningAvatar)->getSkeletonModel(); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 539e109501..b8ccbce962 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -521,8 +521,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent auto startSection = destinationBuffer; auto faceTrackerInfo = reinterpret_cast(destinationBuffer); const auto& blendshapeCoefficients = _headData->getBlendshapeCoefficients(); - - //note: we don't use the blink and average loudness, we just use the numBlendShapes and + // note: we don't use the blink and average loudness, we just use the numBlendShapes and // compute the procedural info on the client side. faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; @@ -1068,7 +1067,6 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { } else { qCWarning(avatars) << "name " << getName() << "key state flag is false"; } - } int numBytesRead = sourceBuffer - startSection; @@ -1121,7 +1119,6 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { } if (hasFaceTrackerInfo) { - //qCWarning(avatars) << "parsing face tracker info "; auto startSection = sourceBuffer; PACKET_READ_CHECK(FaceTrackerInfo, sizeof(AvatarDataPacket::FaceTrackerInfo)); @@ -1129,12 +1126,10 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { int numCoefficients = faceTrackerInfo->numBlendshapeCoefficients; const int coefficientsSize = sizeof(float) * numCoefficients; sourceBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); - + PACKET_READ_CHECK(FaceTrackerCoefficients, coefficientsSize); _headData->_blendshapeCoefficients.resize(numCoefficients); // make sure there's room for the copy! - _headData->_transientBlendshapeCoefficients.resize(numCoefficients); - - //only copy the blendshapes to headData not the procedural face info + //only copy the blendshapes to headData, not the procedural face info memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, coefficientsSize); sourceBuffer += coefficientsSize; diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index f8eca0915e..e8289269d8 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -101,8 +101,6 @@ protected: glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; - //std::atomic _hasProceduralBlinkFaceMovement{ true }; - //std::atomic _hasProceduralEyeFaceMovement{ true }; bool _hasAudioEnabledFaceMovement { true }; bool _hasProceduralBlinkFaceMovement{ true }; bool _hasProceduralEyeFaceMovement{ true }; diff --git a/scripts/developer/DefaultStylizedFemale_Clothed.fst b/scripts/developer/DefaultStylizedFemale_Clothed.fst deleted file mode 100644 index 3e46d6a15c..0000000000 --- a/scripts/developer/DefaultStylizedFemale_Clothed.fst +++ /dev/null @@ -1,139 +0,0 @@ -name = DefaultStylizedFemale_Clothed -type = body+head -scale = 1 -filename = DefaultStylizedFemale_Clothed/DefaultStylizedFemale_Clothed.fbx -texdir = DefaultStylizedFemale_Clothed/textures -joint = jointLean = Spine -joint = jointRightHand = RightHand -joint = jointEyeLeft = LeftEye -joint = jointHead = HeadTop_End -joint = jointNeck = Neck -joint = jointRoot = Hips -joint = jointEyeRight = RightEye -joint = jointLeftHand = LeftHand -freeJoint = LeftArm -freeJoint = LeftForeArm -freeJoint = RightArm -freeJoint = RightForeArm -bs = EyeBlink_L = Blink_Left = 1 -bs = Sneer = Squint_Right = 0.5 -bs = Sneer = Squint_Left = 0.5 -bs = Sneer = NoseScrunch_Right = 0.75 -bs = Sneer = NoseScrunch_Left = 0.75 -bs = ChinLowerRaise = Jaw_Up = 1 -bs = EyeSquint_R = Squint_Right = 1 -bs = MouthSmile_R = Smile_Right = 1 -bs = ChinUpperRaise = UpperLipUp_Right = 0.5 -bs = ChinUpperRaise = UpperLipUp_Left = 0.5 -bs = LipsLowerOpen = LowerLipOut = 1 -bs = LipsLowerDown = LowerLipDown_Right = 0.7 -bs = LipsLowerDown = LowerLipDown_Left = 0.7 -bs = BrowsU_L = BrowsUp_Left = 1 -bs = MouthRight = Midmouth_Right = 1 -bs = MouthDimple_R = Smile_Right = 0.25 -bs = LipsPucker = MouthNarrow_Right = 1 -bs = LipsPucker = MouthNarrow_Left = 1 -bs = Puff = CheekPuff_Right = 1 -bs = Puff = CheekPuff_Left = 1 -bs = JawFwd = JawForeward = 1 -bs = BrowsD_L = BrowsDown_Left = 1 -bs = LipsFunnel = TongueUp = 1 -bs = LipsFunnel = MouthWhistle_NarrowAdjust_Right = 0.5 -bs = LipsFunnel = MouthWhistle_NarrowAdjust_Left = 0.5 -bs = LipsFunnel = MouthNarrow_Right = 1 -bs = LipsFunnel = MouthNarrow_Left = 1 -bs = LipsFunnel = Jaw_Down = 0.36 -bs = LipsFunnel = JawForeward = 0.39 -bs = LipsUpperOpen = UpperLipOut = 1 -bs = EyeSquint_L = Squint_Left = 1 -bs = MouthDimple_L = Smile_Left = 0.25 -bs = LipsLowerClose = LowerLipIn = 1 -bs = MouthFrown_R = Frown_Right = 1 -bs = MouthFrown_L = Frown_Left = 1 -bs = BrowsU_R = BrowsUp_Right = 1 -bs = JawOpen = MouthOpen = 0.7 -bs = JawRight = Jaw_Right = 1 -bs = MouthLeft = Midmouth_Left = 1 -bs = BrowsU_C = BrowsUp_Right = 1 -bs = BrowsU_C = BrowsUp_Left = 1 -bs = LipsUpperUp = UpperLipUp_Right = 0.7 -bs = LipsUpperUp = UpperLipUp_Left = 0.7 -bs = EyeBlink_R = Blink_Right = 1 -bs = EyeOpen_R = EyesWide_Right = 1 -bs = LipsUpperClose = UpperLipIn = 1 -bs = MouthSmile_L = Smile_Left = 1 -bs = EyeOpen_L = EyesWide_Left = 1 -bs = JawLeft = JawRotateY_Left = 0.5 -bs = BrowsD_R = BrowsDown_Right = 1 -jointIndex = RightHandThumb4 = 21 -jointIndex = Neck = 62 -jointIndex = LeftHandIndex4 = 57 -jointIndex = Body = 71 -jointIndex = LeftHandIndex1 = 54 -jointIndex = RightHand = 17 -jointIndex = RightHandMiddle1 = 26 -jointIndex = Spine = 11 -jointIndex = RightHandRing2 = 31 -jointIndex = RightArm = 15 -jointIndex = RightHandPinky2 = 35 -jointIndex = LeftToeBase = 9 -jointIndex = RightHandIndex3 = 24 -jointIndex = RightHandRing1 = 30 -jointIndex = RightHandPinky1 = 34 -jointIndex = RightEye = 66 -jointIndex = LeftHandRing4 = 49 -jointIndex = LeftHandRing2 = 47 -jointIndex = RightHandMiddle2 = 27 -jointIndex = Head = 63 -jointIndex = LeftHandMiddle4 = 53 -jointIndex = LeftLeg = 7 -jointIndex = LeftHandPinky2 = 43 -jointIndex = LeftHandThumb1 = 58 -jointIndex = LeftHandPinky4 = 45 -jointIndex = RightHandIndex1 = 22 -jointIndex = Tops = 67 -jointIndex = Hips = 0 -jointIndex = LeftUpLeg = 6 -jointIndex = RightShoulder = 14 -jointIndex = Spine2 = 13 -jointIndex = RightHandRing4 = 33 -jointIndex = RightHandThumb3 = 20 -jointIndex = RightHandIndex4 = 25 -jointIndex = LeftFoot = 8 -jointIndex = LeftHandRing3 = 48 -jointIndex = LeftHand = 41 -jointIndex = LeftForeArm = 40 -jointIndex = LeftToe_End = 10 -jointIndex = Bottoms = 68 -jointIndex = RightFoot = 3 -jointIndex = LeftHandMiddle2 = 51 -jointIndex = LeftHandThumb3 = 60 -jointIndex = RightHandPinky3 = 36 -jointIndex = LeftEye = 65 -jointIndex = LeftHandIndex2 = 55 -jointIndex = RightHandIndex2 = 23 -jointIndex = LeftHandPinky1 = 42 -jointIndex = LeftHandMiddle3 = 52 -jointIndex = RightHandMiddle4 = 29 -jointIndex = LeftHandThumb2 = 59 -jointIndex = Shoes = 69 -jointIndex = RightHandThumb1 = 18 -jointIndex = RightToe_End = 5 -jointIndex = RightHandThumb2 = 19 -jointIndex = RightUpLeg = 1 -jointIndex = RightLeg = 2 -jointIndex = LeftHandMiddle1 = 50 -jointIndex = LeftHandIndex3 = 56 -jointIndex = LeftHandThumb4 = 61 -jointIndex = RightHandRing3 = 32 -jointIndex = Hair = 70 -jointIndex = Spine1 = 12 -jointIndex = LeftHandRing1 = 46 -jointIndex = LeftArm = 39 -jointIndex = LeftShoulder = 38 -jointIndex = RightForeArm = 16 -jointIndex = HeadTop_End = 64 -jointIndex = RightHandPinky4 = 37 -jointIndex = LeftHandPinky3 = 44 -jointIndex = RightToeBase = 4 -jointIndex = RightHandMiddle3 = 28 diff --git a/scripts/developer/facialExpressions.js b/scripts/developer/facialExpressions.js index 84e3966c4a..37a4f4f796 100644 --- a/scripts/developer/facialExpressions.js +++ b/scripts/developer/facialExpressions.js @@ -1,7 +1,7 @@ // // facialExpressions.js // A script to set different emotions using blend shapes -// +// // Author: Elisa Lupin-Jimenez // Copyright High Fidelity 2018 // @@ -286,7 +286,7 @@ } } for (var blendshape in emotion) { - MyAvatar.setBlendshape(blendshape, + MyAvatar.setBlendshape(blendshape, mixValue(lastEmotionUsed[blendshape], emotion[blendshape], changingEmotionPercentage)); } }); From 67fc7b90b5da4e51c1d9d4e4218a46e206ebde80 Mon Sep 17 00:00:00 2001 From: amantley Date: Thu, 7 Jun 2018 15:43:21 -0700 Subject: [PATCH 19/23] fixed get and set semi nibble to deal with 16 bits --- libraries/avatars-renderer/src/avatars-renderer/Head.cpp | 3 +-- libraries/shared/src/SharedUtil.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 50f20918fa..598bc2b21a 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -59,7 +59,6 @@ void Head::simulate(float deltaTime) { _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); } - //if (!_isFaceTrackerConnected) { if (!_isEyeTrackerConnected) { // Update eye saccades const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; @@ -77,7 +76,7 @@ void Head::simulate(float deltaTime) { } else { _saccade = glm::vec3(); } - + const float BLINK_SPEED = 10.0f; const float BLINK_SPEED_VARIABILITY = 1.0f; const float BLINK_START_VARIABILITY = 0.25f; diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index c0203106c0..bb22a1e753 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -298,11 +298,11 @@ void setAtBit(unsigned char& byte, int bitIndex) { } bool oneAtBit16(unsigned short word, int bitIndex) { - return (word >> (16 - bitIndex) & 1); + return (word >> (15 - bitIndex) & 1); } void setAtBit16(unsigned short& word, int bitIndex) { - word |= (1 << (16 - bitIndex)); + word |= (1 << (15 - bitIndex)); } @@ -313,7 +313,7 @@ void clearAtBit(unsigned char& byte, int bitIndex) { } int getSemiNibbleAt(unsigned short word, int bitIndex) { - return (word >> (6 - bitIndex) & 3); // semi-nibbles store 00, 01, 10, or 11 + return (word >> (14 - bitIndex) & 3); // semi-nibbles store 00, 01, 10, or 11 } int getNthBit(unsigned char byte, int ordinal) { @@ -337,7 +337,7 @@ int getNthBit(unsigned char byte, int ordinal) { void setSemiNibbleAt(unsigned short& word, int bitIndex, int value) { //assert(value <= 3 && value >= 0); - word |= ((value & 3) << (6 - bitIndex)); // semi-nibbles store 00, 01, 10, or 11 + word |= ((value & 3) << (14 - bitIndex)); // semi-nibbles store 00, 01, 10, or 11 } bool isInEnvironment(const char* environment) { From 22f25835569b5d39dc45a1154c28a62cd02d8c11 Mon Sep 17 00:00:00 2001 From: amantley Date: Thu, 7 Jun 2018 16:17:33 -0700 Subject: [PATCH 20/23] removed debug print statement in AvatarData.cpp --- libraries/avatars/src/AvatarData.cpp | 36 ---------------------------- 1 file changed, 36 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index b8ccbce962..b5186ba8f4 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1032,42 +1032,6 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { if (somethingChanged) { _additionalFlagsChanged = usecTimestampNow(); - if (newHasAudioEnabledFaceMovement) { - qCWarning(avatars) << "name " << getName() << "audio enabled flag is true"; - } else { - qCWarning(avatars) << "name " << getName() << "audio enabled flag is false"; - } - if (newHasProceduralEyeFaceMovement) { - qCWarning(avatars) << "name " << getName() << "eye face enabled flag is true"; - } else { - qCWarning(avatars) << "name " << getName() << "eye face flag is false"; - } - if (newHasProceduralBlinkFaceMovement) { - qCWarning(avatars) << "name " << getName() << "blink flag is true"; - } else { - qCWarning(avatars) << "name " << getName() << "blink flag is false"; - } - if (newFaceTrackerConnected) { - qCWarning(avatars) << "name " << getName() << "face tracker flag is true"; - } else { - qCWarning(avatars) << "name " << getName() << "face tracker flag is false"; - } - if (newEyeTrackerConnected) { - qCWarning(avatars) << "name " << getName() << "eye tracker flag is true"; - } else { - qCWarning(avatars) << "name " << getName() << "eye tracker flag is false"; - } - if (newHandState) { - qCWarning(avatars) << "name " << getName() << "hand state flag is true"; - } else { - qCWarning(avatars) << "name " << getName() << "hand state flag is false"; - } - if (newKeyState) { - qCWarning(avatars) << "name " << getName() << "key state flag is true"; - } else { - qCWarning(avatars) << "name " << getName() << "key state flag is false"; - } - } int numBytesRead = sourceBuffer - startSection; _additionalFlagsRate.increment(numBytesRead); From 3f5820266762f9962d9bb5bd91912272005e4f02 Mon Sep 17 00:00:00 2001 From: amantley Date: Thu, 7 Jun 2018 17:41:12 -0700 Subject: [PATCH 21/23] updated the AvatarMixerPacketVersion to ProceduralFaceMovementFlagsAndBlendshapes in PacketHeaders.cpp --- libraries/networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index b69733c18d..c342ecffc5 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -40,7 +40,7 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::AvatarData: case PacketType::BulkAvatarData: case PacketType::KillAvatar: - return static_cast(AvatarMixerPacketVersion::FBXReaderNodeReparenting); + return static_cast(AvatarMixerPacketVersion::ProceduralFaceMovementFlagsAndBlendshapes); case PacketType::MessagesData: return static_cast(MessageDataVersion::TextOrBinaryData); // ICE packets diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 5203a9d178..43495dab07 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -282,7 +282,8 @@ enum class AvatarMixerPacketVersion : PacketVersion { AvatarIdentityLookAtSnapping, UpdatedMannequinDefaultAvatar, AvatarJointDefaultPoseFlags, - FBXReaderNodeReparenting + FBXReaderNodeReparenting, + ProceduralFaceMovementFlagsAndBlendshapes }; enum class DomainConnectRequestVersion : PacketVersion { From b1c578ecaa159536e814a0d528b3328269aa6e1f Mon Sep 17 00:00:00 2001 From: amantley Date: Thu, 7 Jun 2018 22:13:53 -0700 Subject: [PATCH 22/23] fixed white space in head.cpp --- .../src/avatars-renderer/Head.cpp | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 598bc2b21a..5800c1404b 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -95,19 +95,19 @@ void Head::simulate(float deltaTime) { } if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { - // no blinking when brows are raised; blink less with increasing loudness - const float BASE_BLINK_RATE = 15.0f / 60.0f; - const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; - if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * - ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { - _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; - _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; - if (randFloat() < 0.5f) { - _leftEyeBlink = BLINK_START_VARIABILITY; - } else { - _rightEyeBlink = BLINK_START_VARIABILITY; - } + // no blinking when brows are raised; blink less with increasing loudness + const float BASE_BLINK_RATE = 15.0f / 60.0f; + const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; + if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * + ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { + _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + if (randFloat() < 0.5f) { + _leftEyeBlink = BLINK_START_VARIABILITY; + } else { + _rightEyeBlink = BLINK_START_VARIABILITY; } + } } else { _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); From dabcd4c234a58b118e0e5c01be336face48cca25 Mon Sep 17 00:00:00 2001 From: amantley Date: Fri, 8 Jun 2018 10:19:33 -0700 Subject: [PATCH 23/23] fixed bracket spacing in HeadData.h --- libraries/avatars/src/HeadData.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index e8289269d8..f9c4b52139 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -102,8 +102,8 @@ protected: quint64 _lookAtPositionChanged { 0 }; bool _hasAudioEnabledFaceMovement { true }; - bool _hasProceduralBlinkFaceMovement{ true }; - bool _hasProceduralEyeFaceMovement{ true }; + bool _hasProceduralBlinkFaceMovement { true }; + bool _hasProceduralEyeFaceMovement { true }; bool _isFaceTrackerConnected { false }; bool _isEyeTrackerConnected { false }; float _leftEyeBlink { 0.0f };