From 8411e6b03351347e51f4af8176511f42e3823ff7 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 8 Oct 2019 10:42:19 -0700 Subject: [PATCH 01/19] Blendshape script API work * Moved hasScriptedBlendshapes, hasProceduralBlinkFaceMovement, hasProceduralEyeFaceMovement, hasAudioEnabledFaceMovement to AvatarData so they are accessable via agent scripts. * Marked setForceFaceTrackerConnected as depricated. * Updated jsdoc comments --- assignment-client/src/Agent.cpp | 1 - interface/src/avatar/MyAvatar.cpp | 25 ---------- interface/src/avatar/MyAvatar.h | 23 ++------- interface/src/avatar/MyHead.cpp | 4 +- libraries/avatars/src/AvatarData.cpp | 44 +++++++++++++---- libraries/avatars/src/AvatarData.h | 53 +++++++++++++++------ libraries/avatars/src/HeadData.cpp | 7 ++- libraries/avatars/src/HeadData.h | 6 +-- libraries/shared/src/FaceshiftConstants.cpp | 16 +++++-- 9 files changed, 98 insertions(+), 81 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index ee1e21c837..2cad2ca722 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -376,7 +376,6 @@ void Agent::executeScript() { // setup an Avatar for the script to use auto scriptedAvatar = DependencyManager::get(); scriptedAvatar->setID(getSessionUUID()); - scriptedAvatar->setForceFaceTrackerConnected(true); // call model URL setters with empty URLs so our avatar, if user, will have the default models scriptedAvatar->setSkeletonModelURL(QUrl()); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index a9ebd110d1..3c27ef52c2 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -3405,31 +3405,6 @@ bool MyAvatar::shouldRenderHead(const RenderArgs* renderArgs) const { return !defaultMode || (!firstPerson && !insideHead) || (overrideAnim && !insideHead); } -void MyAvatar::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { - if (hasScriptedBlendshapes == _hasScriptedBlendShapes) { - return; - } - if (!hasScriptedBlendshapes) { - // send a forced avatarData update to make sure the script can send neutal blendshapes on unload - // without having to wait for the update loop, make sure _hasScriptedBlendShapes is still true - // before sending the update, or else it won't send the neutal blendshapes to the receiving clients - sendAvatarDataPacket(true); - } - _hasScriptedBlendShapes = hasScriptedBlendshapes; -} - -void MyAvatar::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); -} - -void MyAvatar::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); -} - -void MyAvatar::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); -} - void MyAvatar::setRotationRecenterFilterLength(float length) { const float MINIMUM_ROTATION_RECENTER_FILTER_LENGTH = 0.01f; _rotationRecenterFilterLength = std::max(MINIMUM_ROTATION_RECENTER_FILTER_LENGTH, length); diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 0f139ddbff..acbd0f27a5 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -184,12 +184,6 @@ class MyAvatar : public Avatar { * property value is audioListenerModeCustom. * @property {Quat} customListenOrientation=Quat.IDENTITY - The listening orientation used when the * audioListenerMode property value is audioListenerModeCustom. - * @property {boolean} hasScriptedBlendshapes=false - true to transmit blendshapes over the network. - *

Note: Currently doesn't work. Use {@link MyAvatar.setForceFaceTrackerConnected} instead.

- * @property {boolean} hasProceduralBlinkFaceMovement=true - true if procedural blinking is turned on. - * @property {boolean} hasProceduralEyeFaceMovement=true - true if procedural eye movement is turned on. - * @property {boolean} hasAudioEnabledFaceMovement=true - true to move the mouth blendshapes with voice audio - * when MyAvatar.hasScriptedBlendshapes is enabled. * @property {number} rotationRecenterFilterLength - Configures how quickly the avatar root rotates to recenter its facing * direction to match that of the user's torso based on head and hands orientation. A smaller value makes the * recentering happen more quickly. The minimum value is 0.01. @@ -312,7 +306,10 @@ class MyAvatar : public Avatar { * @borrows Avatar.setAttachmentsVariant as setAttachmentsVariant * @borrows Avatar.updateAvatarEntity as updateAvatarEntity * @borrows Avatar.clearAvatarEntity as clearAvatarEntity - * @borrows Avatar.setForceFaceTrackerConnected as setForceFaceTrackerConnected + * @borrows Avatar.hasScriptedBlendshapes as hasScriptedBlendshapes + * @borrows Avatar.hasProceduralBlinkFaceMovement as hasProceduralBlinkFaceMovement + * @borrows Avatar.hasEyeFaceMovement as hasEyeFaceMovement + * @borrows Avatar.hasAudioEnabledFaceMovement as hasAudioEnabledFaceMovement * @borrows Avatar.setSkeletonModelURL as setSkeletonModelURL * @borrows Avatar.getAttachmentData as getAttachmentData * @borrows Avatar.setAttachmentData as setAttachmentData @@ -359,10 +356,6 @@ class MyAvatar : public Avatar { Q_PROPERTY(AudioListenerMode audioListenerModeCustom READ getAudioListenerModeCustom) Q_PROPERTY(glm::vec3 customListenPosition READ getCustomListenPosition WRITE setCustomListenPosition) Q_PROPERTY(glm::quat customListenOrientation READ getCustomListenOrientation WRITE setCustomListenOrientation) - Q_PROPERTY(bool hasScriptedBlendshapes READ getHasScriptedBlendshapes WRITE setHasScriptedBlendshapes) - Q_PROPERTY(bool hasProceduralBlinkFaceMovement READ getHasProceduralBlinkFaceMovement WRITE setHasProceduralBlinkFaceMovement) - Q_PROPERTY(bool hasProceduralEyeFaceMovement READ getHasProceduralEyeFaceMovement WRITE setHasProceduralEyeFaceMovement) - Q_PROPERTY(bool hasAudioEnabledFaceMovement READ getHasAudioEnabledFaceMovement WRITE setHasAudioEnabledFaceMovement) Q_PROPERTY(float rotationRecenterFilterLength READ getRotationRecenterFilterLength WRITE setRotationRecenterFilterLength) Q_PROPERTY(float rotationThreshold READ getRotationThreshold WRITE setRotationThreshold) Q_PROPERTY(bool enableStepResetRotation READ getEnableStepResetRotation WRITE setEnableStepResetRotation) @@ -2555,14 +2548,6 @@ private: virtual bool shouldRenderHead(const RenderArgs* renderArgs) const override; void setShouldRenderLocally(bool shouldRender) { _shouldRender = shouldRender; setEnableMeshVisible(shouldRender); } bool getShouldRenderLocally() const { return _shouldRender; } - void setHasScriptedBlendshapes(bool hasScriptedBlendshapes); - bool getHasScriptedBlendshapes() const override { return _hasScriptedBlendShapes; } - void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasProceduralBlinkFaceMovement() const override { return _headData->getHasProceduralBlinkFaceMovement(); } - void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralEyeFaceMovement() const override { return _headData->getHasProceduralEyeFaceMovement(); } - void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasAudioEnabledFaceMovement() const override { return _headData->getHasAudioEnabledFaceMovement(); } void setRotationRecenterFilterLength(float length); float getRotationRecenterFilterLength() const { return _rotationRecenterFilterLength; } void setRotationThreshold(float angleRadians); diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index e5c8b71ea2..4705d2d765 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -64,7 +64,9 @@ void MyHead::simulate(float deltaTime) { bool eyeLidsTracked = userInputMapper->getActionStateValid(controller::Action::LEFT_EYE_BLINK) && userInputMapper->getActionStateValid(controller::Action::RIGHT_EYE_BLINK); - setFaceTrackerConnected(eyeLidsTracked); + + setHasScriptedBlendshapes(eyeLidsTracked); + if (eyeLidsTracked) { float leftEyeBlink = userInputMapper->getActionState(controller::Action::LEFT_EYE_BLINK); float rightEyeBlink = userInputMapper->getActionState(controller::Action::RIGHT_EYE_BLINK); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 710bfb8d2a..520241d020 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -110,7 +110,7 @@ AvatarData::AvatarData() : _targetScale(1.0f), _handState(0), _keyState(NO_KEY_DOWN), - _forceFaceTrackerConnected(false), + _hasScriptedBlendshapes(false), _headData(NULL), _errorLogExpiry(0), _owningAvatarMixer(), @@ -154,6 +154,32 @@ float AvatarData::getDomainLimitedScale() const { } } + +void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { + if (hasScriptedBlendshapes == _hasScriptedBlendshapes) { + return; + } + if (!hasScriptedBlendshapes) { + // send a forced avatarData update to make sure the script can send neutal blendshapes on unload + // without having to wait for the update loop, make sure _hasScriptedBlendShapes is still true + // before sending the update, or else it won't send the neutal blendshapes to the receiving clients + sendAvatarDataPacket(true); + } + _hasScriptedBlendshapes = hasScriptedBlendshapes; +} + +void AvatarData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { + _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); +} + +void AvatarData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { + _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); +} + +void AvatarData::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { + _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); +} + void AvatarData::setDomainMinimumHeight(float domainMinimumHeight) { _domainMinimumHeight = glm::clamp(domainMinimumHeight, MIN_AVATAR_HEIGHT, MAX_AVATAR_HEIGHT); } @@ -206,8 +232,8 @@ void AvatarData::lazyInitHeadData() const { if (!_headData) { _headData = new HeadData(const_cast(this)); } - if (_forceFaceTrackerConnected) { - _headData->_isFaceTrackerConnected = true; + if (_hasScriptedBlendshapes) { + _headData->_hasScriptedBlendshapes = true; } } @@ -338,7 +364,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent tranlationChangedSince(lastSentTime) || parentInfoChangedSince(lastSentTime)); hasHandControllers = _controllerLeftHandMatrixCache.isValid() || _controllerRightHandMatrixCache.isValid(); - hasFaceTrackerInfo = !dropFaceTracking && (hasFaceTracker() || getHasScriptedBlendshapes()) && + hasFaceTrackerInfo = !dropFaceTracking && getHasScriptedBlendshapes() && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); hasJointData = !sendMinimum; hasJointDefaultPoseFlags = hasJointData; @@ -529,8 +555,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent setAtBit16(flags, HAND_STATE_FINGER_POINTING_BIT); } // face tracker state - if (_headData->_isFaceTrackerConnected) { - setAtBit16(flags, IS_FACE_TRACKER_CONNECTED); + if (_headData->_hasScriptedBlendshapes) { + setAtBit16(flags, HAS_SCRIPTED_BLENDSHAPES); } // eye tracker state if (!_headData->_hasProceduralEyeMovement) { @@ -1150,7 +1176,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { auto newHandState = getSemiNibbleAt(bitItems, HAND_STATE_START_BIT) + (oneAtBit16(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0); - auto newFaceTrackerConnected = oneAtBit16(bitItems, IS_FACE_TRACKER_CONNECTED); + auto newHasScriptedBlendshapes = oneAtBit16(bitItems, HAS_SCRIPTED_BLENDSHAPES); auto newHasntProceduralEyeMovement = oneAtBit16(bitItems, IS_EYE_TRACKER_CONNECTED); auto newHasAudioEnabledFaceMovement = oneAtBit16(bitItems, AUDIO_ENABLED_FACE_MOVEMENT); @@ -1161,7 +1187,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { bool keyStateChanged = (_keyState != newKeyState); bool handStateChanged = (_handState != newHandState); - bool faceStateChanged = (_headData->_isFaceTrackerConnected != newFaceTrackerConnected); + bool faceStateChanged = (_headData->_hasScriptedBlendshapes != newHasScriptedBlendshapes); bool eyeStateChanged = (_headData->_hasProceduralEyeMovement == newHasntProceduralEyeMovement); bool audioEnableFaceMovementChanged = (_headData->getHasAudioEnabledFaceMovement() != newHasAudioEnabledFaceMovement); bool proceduralEyeFaceMovementChanged = (_headData->getHasProceduralEyeFaceMovement() != newHasProceduralEyeFaceMovement); @@ -1174,7 +1200,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _keyState = newKeyState; _handState = newHandState; - _headData->_isFaceTrackerConnected = newFaceTrackerConnected; + _headData->_hasScriptedBlendshapes = newHasScriptedBlendshapes; _headData->setHasProceduralEyeMovement(!newHasntProceduralEyeMovement); _headData->setHasAudioEnabledFaceMovement(newHasAudioEnabledFaceMovement); _headData->setHasProceduralEyeFaceMovement(newHasProceduralEyeFaceMovement); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index df0783ef4b..6bed5fa9db 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -104,12 +104,12 @@ const quint32 AVATAR_MOTION_SCRIPTABLE_BITS = // Procedural Collide with other avatars is enabled 12th bit // Procedural Has Hero Priority is enabled 13th bit -const int KEY_STATE_START_BIT = 0; // 1st and 2nd bits -const int HAND_STATE_START_BIT = 2; // 3rd and 4th bits -const int IS_FACE_TRACKER_CONNECTED = 4; // 5th bit +const int KEY_STATE_START_BIT = 0; // 1st and 2nd bits (UNUSED) +const int HAND_STATE_START_BIT = 2; // 3rd and 4th bits (UNUSED) +const int HAS_SCRIPTED_BLENDSHAPES = 4; // 5th bit const int IS_EYE_TRACKER_CONNECTED = 5; // 6th bit (was CHAT_CIRCLING) const int HAS_REFERENTIAL = 6; // 7th bit -const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit +const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit (UNUSED) const int AUDIO_ENABLED_FACE_MOVEMENT = 8; // 9th bit const int PROCEDURAL_EYE_FACE_MOVEMENT = 9; // 10th bit const int PROCEDURAL_BLINK_FACE_MOVEMENT = 10; // 11th bit @@ -325,7 +325,7 @@ namespace AvatarDataPacket { // variable length structure follows - // only present if IS_FACE_TRACKER_CONNECTED flag is set in AvatarInfo.flags + // only present if HAS_SCRIPTED_BLENDSHAPES flag is set in AvatarInfo.flags PACKED_BEGIN struct FaceTrackerInfo { float leftEyeBlink; float rightEyeBlink; @@ -534,6 +534,19 @@ class AvatarData : public QObject, public SpatiallyNestable { * size in the virtual world. Read-only. * @property {boolean} hasPriority - true if the avatar is in a "hero" zone, false if it isn't. * Read-only. + * @property {boolean} hasScriptedBlendshapes=false - Set this to true before using the {@link MyAvatar.setBlendshape} method, + * after you no longer want scripted control over the blendshapes set to back to false.
NOTE: this property will + * automatically become true if the Controller system has valid facial blendshape actions. + * @property {boolean} hasProceduralBlinkFaceMovement=true - By default avatars will blink automatically by animating facial + * blendshapes. Set this property to false to disable this automatic blinking. This can be useful if you + * wish to fully control the blink facial blendshapes via the {@link MyAvatar.setBlendshape} method. + * @property {boolean} hasProceduralEyeFaceMovement=true - By default the avatar eye facial blendshapes will be adjusted + * automatically as the eyes move. This will prevent the iris is never obscured by the upper or lower lids. Set this + * property to false to disable this automatic movement. This can be useful if you wish to fully control + * the eye blendshapes via the {@link MyAvatar.setBlendshape} method. + * @property {boolean} hasAudioEnabledFaceMovement=true - By default the avatar mouth blendshapes will animate based on + * the microphone audio. Set this property to false to disable that animaiton. This can be useful if you + * wish to fully control the blink facial blendshapes via the {@link MyAvatar.setBlendshape} method. */ Q_PROPERTY(glm::vec3 position READ getWorldPosition WRITE setPositionViaScript) Q_PROPERTY(float scale READ getDomainLimitedScale WRITE setTargetScale) @@ -575,6 +588,11 @@ class AvatarData : public QObject, public SpatiallyNestable { Q_PROPERTY(bool hasPriority READ getHasPriority) + Q_PROPERTY(bool hasScriptedBlendshapes READ getHasScriptedBlendshapes WRITE setHasScriptedBlendshapes) + Q_PROPERTY(bool hasProceduralBlinkFaceMovement READ getHasProceduralBlinkFaceMovement WRITE setHasProceduralBlinkFaceMovement) + Q_PROPERTY(bool hasProceduralEyeFaceMovement READ getHasProceduralEyeFaceMovement WRITE setHasProceduralEyeFaceMovement) + Q_PROPERTY(bool hasAudioEnabledFaceMovement READ getHasAudioEnabledFaceMovement WRITE setHasAudioEnabledFaceMovement) + public: virtual QString getName() const override { return QString("Avatar:") + _displayName; } @@ -684,10 +702,14 @@ public: float getDomainLimitedScale() const; - virtual bool getHasScriptedBlendshapes() const { return false; } - virtual bool getHasProceduralBlinkFaceMovement() const { return true; } - virtual bool getHasProceduralEyeFaceMovement() const { return true; } - virtual bool getHasAudioEnabledFaceMovement() const { return false; } + void setHasScriptedBlendshapes(bool hasScriptedBlendshapes); + bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } + void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); + bool getHasProceduralBlinkFaceMovement() const { return _headData->getHasProceduralBlinkFaceMovement(); } + void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); + bool getHasProceduralEyeFaceMovement() const { return _headData->getHasProceduralEyeFaceMovement(); } + void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); + bool getHasAudioEnabledFaceMovement() const { return _headData->getHasAudioEnabledFaceMovement(); } /**jsdoc * Gets the minimum scale allowed for this avatar in the current domain. @@ -1111,13 +1133,14 @@ public: /**jsdoc * Sets the value of a blendshape to animate your avatar's face. To enable other users to see the resulting animation of - * your avatar's face, use {@link Avatar.setForceFaceTrackerConnected} or {@link MyAvatar.setForceFaceTrackerConnected}. + * your avatar's face, set {@link Avatar.hasScriptedBlendshapes} to true while using this API and back to false when your + * animation is complete. * @function Avatar.setBlendshape * @param {string} name - The name of the blendshape, per the * {@link https://docs.highfidelity.com/create/avatars/avatar-standards.html#blendshapes Avatar Standards}. * @param {number} value - A value between 0.0 and 1.0. * @example Open your avatar's mouth wide. - * MyAvatar.setForceFaceTrackerConnected(true); + * MyAvatar.hasScriptedBlendshapes = true; * MyAvatar.setBlendshape("JawOpen", 1.0); * * // Note: If using from the Avatar API, replace "MyAvatar" with "Avatar". @@ -1163,15 +1186,16 @@ public: */ Q_INVOKABLE virtual void clearAvatarEntity(const QUuid& entityID, bool requiresRemovalFromTree = true); - /**jsdoc + *

Deprecated: This method is deprecated and will be removed.

+ * Use Avatar.hasScriptedBlendshapes property instead. * Enables blendshapes set using {@link Avatar.setBlendshape} or {@link MyAvatar.setBlendshape} to be transmitted to other * users so that they can see the animation of your avatar's face. * @function Avatar.setForceFaceTrackerConnected * @param {boolean} connected - true to enable blendshape changes to be transmitted to other users, * false to disable. */ - Q_INVOKABLE void setForceFaceTrackerConnected(bool connected) { _forceFaceTrackerConnected = connected; } + Q_INVOKABLE void setForceFaceTrackerConnected(bool connected) { setHasScriptedBlendshapes(connected); } // key state void setKeyState(KeyState s) { _keyState = s; } @@ -1660,7 +1684,6 @@ protected: bool faceTrackerInfoChangedSince(quint64 time) const { return true; } // FIXME bool hasParent() const { return !getParentID().isNull(); } - bool hasFaceTracker() const { return _headData ? _headData->_isFaceTrackerConnected : false; } QByteArray packSkeletonData() const; QByteArray packSkeletonModelURL() const; @@ -1693,7 +1716,7 @@ protected: // key state KeyState _keyState; - bool _forceFaceTrackerConnected; + bool _hasScriptedBlendshapes; bool _hasNewJointData { true }; // set in AvatarData, cleared in Avatar mutable HeadData* _headData { nullptr }; diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index c86e534929..0e0dd9887f 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -206,8 +206,7 @@ void HeadData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement } bool HeadData::getHasProceduralBlinkFaceMovement() const { - // return _hasProceduralBlinkFaceMovement; - return _hasProceduralBlinkFaceMovement && !_isFaceTrackerConnected; + return _hasProceduralBlinkFaceMovement; } void HeadData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { @@ -230,6 +229,6 @@ void HeadData::setHasProceduralEyeMovement(bool hasProceduralEyeMovement) { _hasProceduralEyeMovement = hasProceduralEyeMovement; } -void HeadData::setFaceTrackerConnected(bool value) { - _isFaceTrackerConnected = value; +void HeadData::setHasScriptedBlendshapes(bool value) { + _hasScriptedBlendshapes = value; } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index dc5aaf2595..e5dd158505 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -81,8 +81,8 @@ public: bool getHasProceduralEyeMovement() const; void setHasProceduralEyeMovement(bool hasProceduralEyeMovement); - void setFaceTrackerConnected(bool value); - bool getFaceTrackerConnected() const { return _isFaceTrackerConnected; } + void setHasScriptedBlendshapes(bool value); + bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } friend class AvatarData; @@ -103,7 +103,7 @@ protected: bool _hasProceduralEyeFaceMovement { true }; bool _hasProceduralEyeMovement { true }; - bool _isFaceTrackerConnected { false }; + bool _hasScriptedBlendshapes { false }; float _leftEyeBlink { 0.0f }; float _rightEyeBlink { 0.0f }; diff --git a/libraries/shared/src/FaceshiftConstants.cpp b/libraries/shared/src/FaceshiftConstants.cpp index 0d6f718e49..b847837a7a 100644 --- a/libraries/shared/src/FaceshiftConstants.cpp +++ b/libraries/shared/src/FaceshiftConstants.cpp @@ -34,7 +34,7 @@ const char* FACESHIFT_BLENDSHAPES[] = { "JawFwd", "JawLeft", "JawOpen", - "JawChew", + "JawChew", // legacy not in ARKit "JawRight", "MouthLeft", "MouthRight", @@ -48,21 +48,29 @@ const char* FACESHIFT_BLENDSHAPES[] = { "LipsStretch_R", "LipsUpperClose", "LipsLowerClose", - "LipsUpperUp", - "LipsLowerDown", + "LipsUpperUp", // legacy, split in ARKit + "LipsLowerDown", // legacy, split in ARKit "LipsUpperOpen", "LipsLowerOpen", "LipsFunnel", "LipsPucker", "ChinLowerRaise", "ChinUpperRaise", - "Sneer", + "Sneer", // legacy, split in ARKit "Puff", "CheekSquint_L", "CheekSquint_R", "" }; +// new in ARKit +// LipsTogether +// MouthPressLeft +// MouthPressRight +// MouthShrugLower +// MouthShrugUpper +// TongueOut + const int NUM_FACESHIFT_BLENDSHAPES = sizeof(FACESHIFT_BLENDSHAPES) / sizeof(char*); const int EYE_BLINK_L_INDEX = 0; From da0911e01b8a88b630513788601cee15650c31be Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 8 Oct 2019 16:08:29 -0700 Subject: [PATCH 02/19] Delete legacy blendshape code Move FaceshiftConstants to BlendshapeConstants. Delete FaceTracker and DdeFaceTracker classes. Delete old facetracker menu and preferences --- interface/CMakeLists.txt | 4 +- interface/src/Application.cpp | 113 +-- interface/src/Application.h | 9 - interface/src/Menu.cpp | 42 -- interface/src/avatar/MyAvatar.cpp | 73 +- interface/src/avatar/MyAvatar.h | 4 +- interface/src/avatar/MyHead.cpp | 4 +- interface/src/devices/DdeFaceTracker.cpp | 686 ------------------ interface/src/devices/DdeFaceTracker.h | 181 ----- interface/src/ui/AvatarInputs.cpp | 10 - interface/src/ui/AvatarInputs.h | 33 - interface/src/ui/PreferencesDialog.cpp | 17 - libraries/avatars-renderer/CMakeLists.txt | 2 +- .../src/avatars-renderer/Head.cpp | 19 +- libraries/avatars/src/HeadData.cpp | 2 +- libraries/avatars/src/HeadData.h | 2 +- libraries/fbx/src/FBXSerializer.cpp | 2 +- libraries/fbx/src/GLTFSerializer.cpp | 2 +- ...tConstants.cpp => BlendshapeConstants.cpp} | 14 +- libraries/shared/src/BlendshapeConstants.h | 76 ++ libraries/shared/src/FaceshiftConstants.h | 25 - libraries/trackers/CMakeLists.txt | 7 - .../trackers/src/trackers/FaceTracker.cpp | 133 ---- libraries/trackers/src/trackers/FaceTracker.h | 131 ---- libraries/trackers/src/trackers/Logging.cpp | 11 - libraries/trackers/src/trackers/Logging.h | 16 - tools/jsdoc/plugins/hifi.js | 2 - 27 files changed, 110 insertions(+), 1510 deletions(-) delete mode 100644 interface/src/devices/DdeFaceTracker.cpp delete mode 100644 interface/src/devices/DdeFaceTracker.h rename libraries/shared/src/{FaceshiftConstants.cpp => BlendshapeConstants.cpp} (84%) create mode 100644 libraries/shared/src/BlendshapeConstants.h delete mode 100644 libraries/shared/src/FaceshiftConstants.h delete mode 100644 libraries/trackers/CMakeLists.txt delete mode 100644 libraries/trackers/src/trackers/FaceTracker.cpp delete mode 100644 libraries/trackers/src/trackers/FaceTracker.h delete mode 100644 libraries/trackers/src/trackers/Logging.cpp delete mode 100644 libraries/trackers/src/trackers/Logging.h diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index bcd3f269e8..9030666609 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -211,10 +211,10 @@ endif() link_hifi_libraries( shared workload task octree ktx gpu gl procedural graphics graphics-scripting render pointers recording hfm fbx networking material-networking - model-networking model-baker entities avatars trackers + model-networking model-baker entities avatars audio audio-client animation script-engine physics render-utils entities-renderer avatars-renderer ui qml auto-updater midi - controllers plugins image trackers platform + controllers plugins image platform ui-plugins display-plugins input-plugins # Platform specific GL libraries ${PLATFORM_GL_BACKEND} diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 92bc54d43f..677137148c 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -170,7 +170,6 @@ #include "avatar/MyCharacterController.h" #include "CrashRecoveryHandler.h" #include "CrashHandler.h" -#include "devices/DdeFaceTracker.h" #include "DiscoverabilityManager.h" #include "GLCanvas.h" #include "InterfaceDynamicFactory.h" @@ -887,11 +886,6 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); - -#ifdef HAVE_DDE - DependencyManager::set(); -#endif - DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); @@ -1067,7 +1061,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo _lastSendDownstreamAudioStats(usecTimestampNow()), _notifiedPacketVersionMismatchThisDomain(false), _maxOctreePPS(maxOctreePacketsPerSecond.get()), - _lastFaceTrackerUpdate(0), _snapshotSound(nullptr), _sampleSound(nullptr) { @@ -2014,13 +2007,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo this->installEventFilter(this); - -#ifdef HAVE_DDE - auto ddeTracker = DependencyManager::get(); - ddeTracker->init(); - connect(ddeTracker.data(), &FaceTracker::muteToggled, this, &Application::faceTrackerMuteToggled); -#endif - // If launched from Steam, let it handle updates const QString HIFI_NO_UPDATER_COMMAND_LINE_KEY = "--no-updater"; bool noUpdater = arguments().indexOf(HIFI_NO_UPDATER_COMMAND_LINE_KEY) != -1; @@ -2762,9 +2748,6 @@ void Application::cleanupBeforeQuit() { } // Stop third party processes so that they're not left running in the event of a subsequent shutdown crash. -#ifdef HAVE_DDE - DependencyManager::get()->setEnabled(false); -#endif AnimDebugDraw::getInstance().shutdown(); // FIXME: once we move to shared pointer for the INputDevice we shoud remove this naked delete: @@ -2835,10 +2818,6 @@ void Application::cleanupBeforeQuit() { _window->saveGeometry(); // Destroy third party processes after scripts have finished using them. -#ifdef HAVE_DDE - DependencyManager::destroy(); -#endif - DependencyManager::destroy(); // Must be destroyed before TabletScriptingInterface // stop QML @@ -3457,9 +3436,6 @@ void Application::onDesktopRootContextCreated(QQmlContext* surfaceContext) { surfaceContext->setContextProperty("AccountServices", AccountServicesScriptingInterface::getInstance()); surfaceContext->setContextProperty("DialogsManager", _dialogsManagerScriptingInterface); -#ifdef HAVE_DDE - surfaceContext->setContextProperty("FaceTracker", DependencyManager::get().data()); -#endif surfaceContext->setContextProperty("AvatarManager", DependencyManager::get().data()); surfaceContext->setContextProperty("LODManager", DependencyManager::get().data()); surfaceContext->setContextProperty("HMD", DependencyManager::get().data()); @@ -3723,16 +3699,6 @@ void Application::runTests() { runUnitTests(); } -void Application::faceTrackerMuteToggled() { - - QAction* muteAction = Menu::getInstance()->getActionForOption(MenuOption::MuteFaceTracking); - Q_CHECK_PTR(muteAction); - bool isMuted = getSelectedFaceTracker()->isMuted(); - muteAction->setChecked(isMuted); - getSelectedFaceTracker()->setEnabled(!isMuted); - Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setEnabled(!isMuted); -} - void Application::setFieldOfView(float fov) { if (fov != _fieldOfView.get()) { _fieldOfView.set(fov); @@ -5307,43 +5273,6 @@ ivec2 Application::getMouse() const { return getApplicationCompositor().getReticlePosition(); } -FaceTracker* Application::getActiveFaceTracker() { -#ifdef HAVE_DDE - auto dde = DependencyManager::get(); - - if (dde && dde->isActive()) { - return static_cast(dde.data()); - } -#endif - - return nullptr; -} - -FaceTracker* Application::getSelectedFaceTracker() { - FaceTracker* faceTracker = nullptr; -#ifdef HAVE_DDE - if (Menu::getInstance()->isOptionChecked(MenuOption::UseCamera)) { - faceTracker = DependencyManager::get().data(); - } -#endif - return faceTracker; -} - -void Application::setActiveFaceTracker() const { -#ifdef HAVE_DDE - bool isMuted = Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking); - bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::UseCamera); - Menu::getInstance()->getActionForOption(MenuOption::BinaryEyelidControl)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::CoupleEyelids)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::VelocityFilter)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setVisible(isUsingDDE); - auto ddeTracker = DependencyManager::get(); - ddeTracker->setIsMuted(isMuted); - ddeTracker->setEnabled(isUsingDDE && !isMuted); -#endif -} - bool Application::exportEntities(const QString& filename, const QVector& entityIDs, const glm::vec3* givenOffset) { @@ -5827,8 +5756,7 @@ void Application::updateMyAvatarLookAtPosition() { PerformanceWarning warn(showWarnings, "Application::updateMyAvatarLookAtPosition()"); auto myAvatar = getMyAvatar(); - FaceTracker* faceTracker = getActiveFaceTracker(); - myAvatar->updateLookAtPosition(faceTracker, _myCamera); + myAvatar->updateLookAtPosition(_myCamera); } void Application::updateThreads(float deltaTime) { @@ -6254,37 +6182,6 @@ void Application::update(float deltaTime) { auto myAvatar = getMyAvatar(); { PerformanceTimer perfTimer("devices"); - - FaceTracker* tracker = getSelectedFaceTracker(); - if (tracker && Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking) != tracker->isMuted()) { - tracker->toggleMute(); - } - - tracker = getActiveFaceTracker(); - if (tracker && !tracker->isMuted()) { - tracker->update(deltaTime); - - // Auto-mute microphone after losing face tracking? - if (tracker->isTracking()) { - _lastFaceTrackerUpdate = usecTimestampNow(); - } else { - const quint64 MUTE_MICROPHONE_AFTER_USECS = 5000000; //5 secs - Menu* menu = Menu::getInstance(); - auto audioClient = DependencyManager::get(); - if (menu->isOptionChecked(MenuOption::AutoMuteAudio) && !audioClient->isMuted()) { - if (_lastFaceTrackerUpdate > 0 - && ((usecTimestampNow() - _lastFaceTrackerUpdate) > MUTE_MICROPHONE_AFTER_USECS)) { - audioClient->setMuted(true); - _lastFaceTrackerUpdate = 0; - } - } else { - _lastFaceTrackerUpdate = 0; - } - } - } else { - _lastFaceTrackerUpdate = 0; - } - auto userInputMapper = DependencyManager::get(); controller::HmdAvatarAlignmentType hmdAvatarAlignmentType; @@ -7080,10 +6977,6 @@ void Application::copyDisplayViewFrustum(ViewFrustum& viewOut) const { // feature. However, we still use this to reset face trackers, eye trackers, audio and to optionally re-load the avatar // rig and animations from scratch. void Application::resetSensors(bool andReload) { -#ifdef HAVE_DDE - DependencyManager::get()->reset(); -#endif - _overlayConductor.centerUI(); getActiveDisplayPlugin()->resetSensors(); getMyAvatar()->reset(true, andReload); @@ -7485,10 +7378,6 @@ void Application::registerScriptEngineWithApplicationServices(const ScriptEngine scriptEngine->registerGlobalObject("AccountServices", AccountServicesScriptingInterface::getInstance()); qScriptRegisterMetaType(scriptEngine.data(), DownloadInfoResultToScriptValue, DownloadInfoResultFromScriptValue); -#ifdef HAVE_DDE - scriptEngine->registerGlobalObject("FaceTracker", DependencyManager::get().data()); -#endif - scriptEngine->registerGlobalObject("AvatarManager", DependencyManager::get().data()); scriptEngine->registerGlobalObject("LODManager", DependencyManager::get().data()); diff --git a/interface/src/Application.h b/interface/src/Application.h index e3334d12d6..80b10697b8 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -81,7 +81,6 @@ #include "VisionSqueeze.h" class GLCanvas; -class FaceTracker; class MainWindow; class AssetUpload; class CompositorHelper; @@ -191,9 +190,6 @@ public: ivec2 getMouse() const; - FaceTracker* getActiveFaceTracker(); - FaceTracker* getSelectedFaceTracker(); - ApplicationOverlay& getApplicationOverlay() { return _applicationOverlay; } const ApplicationOverlay& getApplicationOverlay() const { return _applicationOverlay; } CompositorHelper& getApplicationCompositor() const; @@ -423,7 +419,6 @@ public slots: static void packageModel(); void resetSensors(bool andReload = false); - void setActiveFaceTracker() const; void hmdVisibleChanged(bool visible); @@ -497,8 +492,6 @@ private slots: void resettingDomain(); - void faceTrackerMuteToggled(); - void activeChanged(Qt::ApplicationState state); void windowMinimizedChanged(bool minimized); @@ -736,8 +729,6 @@ private: PerformanceManager _performanceManager; RefreshRateManager _refreshRateManager; - quint64 _lastFaceTrackerUpdate; - GameWorkload _gameWorkload; GraphicsEngine _graphicsEngine; diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 5be4db46a0..c0df4ff159 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -37,7 +37,6 @@ #include "avatar/AvatarManager.h" #include "avatar/AvatarPackager.h" #include "AvatarBookmarks.h" -#include "devices/DdeFaceTracker.h" #include "MainWindow.h" #include "render/DrawStatus.h" #include "scripting/MenuScriptingInterface.h" @@ -493,47 +492,6 @@ Menu::Menu() { // Developer > Avatar >>> MenuWrapper* avatarDebugMenu = developerMenu->addMenu("Avatar"); - // Developer > Avatar > Face Tracking - MenuWrapper* faceTrackingMenu = avatarDebugMenu->addMenu("Face Tracking"); - { - QActionGroup* faceTrackerGroup = new QActionGroup(avatarDebugMenu); - - bool defaultNoFaceTracking = true; -#ifdef HAVE_DDE - defaultNoFaceTracking = false; -#endif - QAction* noFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::NoFaceTracking, - 0, defaultNoFaceTracking, - qApp, SLOT(setActiveFaceTracker())); - faceTrackerGroup->addAction(noFaceTracker); - -#ifdef HAVE_DDE - QAction* ddeFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseCamera, - 0, true, - qApp, SLOT(setActiveFaceTracker())); - faceTrackerGroup->addAction(ddeFaceTracker); -#endif - } -#ifdef HAVE_DDE - faceTrackingMenu->addSeparator(); - QAction* binaryEyelidControl = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::BinaryEyelidControl, 0, true); - binaryEyelidControl->setVisible(true); // DDE face tracking is on by default - QAction* coupleEyelids = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::CoupleEyelids, 0, true); - coupleEyelids->setVisible(true); // DDE face tracking is on by default - QAction* useAudioForMouth = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseAudioForMouth, 0, true); - useAudioForMouth->setVisible(true); // DDE face tracking is on by default - QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::VelocityFilter, 0, true); - ddeFiltering->setVisible(true); // DDE face tracking is on by default - QAction* ddeCalibrate = addActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::CalibrateCamera, 0, - DependencyManager::get().data(), SLOT(calibrate())); - ddeCalibrate->setVisible(true); // DDE face tracking is on by default - faceTrackingMenu->addSeparator(); - addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::MuteFaceTracking, - [](bool mute) { FaceTracker::setIsMuted(mute); }, - Qt::CTRL | Qt::SHIFT | Qt::Key_F, FaceTracker::isMuted()); - addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::AutoMuteAudio, 0, false); -#endif - action = addCheckableActionToQMenuAndActionHash(avatarDebugMenu, MenuOption::AvatarReceiveStats, 0, false); connect(action, &QAction::triggered, [this]{ Avatar::setShowReceiveStats(isOptionChecked(MenuOption::AvatarReceiveStats)); }); action = addCheckableActionToQMenuAndActionHash(avatarDebugMenu, MenuOption::ShowBoundingCollisionShapes, 0, false); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 3c27ef52c2..fb42f89048 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -48,7 +48,6 @@ #include #include #include -#include #include #include @@ -749,7 +748,6 @@ void MyAvatar::update(float deltaTime) { Head* head = getHead(); head->relax(deltaTime); - updateFromTrackers(deltaTime); if (getIsInWalkingState() && glm::length(getControllerPoseInAvatarFrame(controller::Action::HEAD).getVelocity()) < DEFAULT_AVATAR_WALK_SPEED_THRESHOLD) { setIsInWalkingState(false); @@ -1140,60 +1138,6 @@ void MyAvatar::updateSensorToWorldMatrix() { } -// Update avatar head rotation with sensor data -void MyAvatar::updateFromTrackers(float deltaTime) { - glm::vec3 estimatedRotation; - - bool hasHead = getControllerPoseInAvatarFrame(controller::Action::HEAD).isValid(); - bool playing = DependencyManager::get()->isPlaying(); - if (hasHead && playing) { - return; - } - - FaceTracker* tracker = qApp->getActiveFaceTracker(); - bool inFacetracker = tracker && !FaceTracker::isMuted(); - - if (inFacetracker) { - estimatedRotation = glm::degrees(safeEulerAngles(tracker->getHeadRotation())); - } - - // Rotate the body if the head is turned beyond the screen - if (Menu::getInstance()->isOptionChecked(MenuOption::TurnWithHead)) { - const float TRACKER_YAW_TURN_SENSITIVITY = 0.5f; - const float TRACKER_MIN_YAW_TURN = 15.0f; - const float TRACKER_MAX_YAW_TURN = 50.0f; - if ( (fabs(estimatedRotation.y) > TRACKER_MIN_YAW_TURN) && - (fabs(estimatedRotation.y) < TRACKER_MAX_YAW_TURN) ) { - if (estimatedRotation.y > 0.0f) { - _bodyYawDelta += (estimatedRotation.y - TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; - } else { - _bodyYawDelta += (estimatedRotation.y + TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; - } - } - } - - // Set the rotation of the avatar's head (as seen by others, not affecting view frustum) - // to be scaled such that when the user's physical head is pointing at edge of screen, the - // avatar head is at the edge of the in-world view frustum. So while a real person may move - // their head only 30 degrees or so, this may correspond to a 90 degree field of view. - // Note that roll is magnified by a constant because it is not related to field of view. - - - Head* head = getHead(); - if (hasHead || playing) { - head->setDeltaPitch(estimatedRotation.x); - head->setDeltaYaw(estimatedRotation.y); - head->setDeltaRoll(estimatedRotation.z); - } else { - ViewFrustum viewFrustum; - qApp->copyViewFrustum(viewFrustum); - float magnifyFieldOfView = viewFrustum.getFieldOfView() / _realWorldFieldOfView.get(); - head->setDeltaPitch(estimatedRotation.x * magnifyFieldOfView); - head->setDeltaYaw(estimatedRotation.y * magnifyFieldOfView); - head->setDeltaRoll(estimatedRotation.z); - } -} - glm::vec3 MyAvatar::getLeftHandPosition() const { auto pose = getControllerPoseInAvatarFrame(controller::Action::LEFT_HAND); return pose.isValid() ? pose.getTranslation() : glm::vec3(0.0f); @@ -6585,7 +6529,7 @@ bool MyAvatar::getIsJointOverridden(int jointIndex) const { return _skeletonModel->getIsJointOverridden(jointIndex); } -void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) { +void MyAvatar::updateLookAtPosition(Camera& myCamera) { updateLookAtTargetAvatar(); @@ -6681,21 +6625,6 @@ void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) (getHead()->getFinalOrientationInWorldFrame() * glm::vec3(0.0f, 0.0f, -TREE_SCALE)); } } - - // Deflect the eyes a bit to match the detected gaze from the face tracker if active. - if (faceTracker && !faceTracker->isMuted()) { - float eyePitch = faceTracker->getEstimatedEyePitch(); - float eyeYaw = faceTracker->getEstimatedEyeYaw(); - const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f; - glm::vec3 origin = myHead->getEyePosition(); - float deflection = faceTracker->getEyeDeflection(); - if (isLookingAtSomeone) { - deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT; - } - lookAtSpot = origin + myCamera.getOrientation() * glm::quat(glm::radians(glm::vec3( - eyePitch * deflection, eyeYaw * deflection, 0.0f))) * - glm::inverse(myCamera.getOrientation()) * (lookAtSpot - origin); - } } } diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index acbd0f27a5..d67a2567f8 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -34,7 +34,6 @@ #include "AtRestDetector.h" #include "MyCharacterController.h" #include "RingBufferHistory.h" -#include "devices/DdeFaceTracker.h" class AvatarActionHold; class ModelItemID; @@ -1899,7 +1898,7 @@ public: bool getFlowActive() const; bool getNetworkGraphActive() const; - void updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera); + void updateLookAtPosition(Camera& myCamera); // sets the reaction enabled and triggered parameters of the passed in params // also clears internal reaction triggers @@ -2542,7 +2541,6 @@ private: virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking) override; void simulate(float deltaTime, bool inView) override; - void updateFromTrackers(float deltaTime); void saveAvatarUrl(); virtual void render(RenderArgs* renderArgs) override; virtual bool shouldRenderHead(const RenderArgs* renderArgs) const override; diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 4705d2d765..1c6e4690dd 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -14,10 +14,8 @@ #include #include #include -#include -#include +#include -#include "devices/DdeFaceTracker.h" #include "Application.h" #include "MyAvatar.h" diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp deleted file mode 100644 index b9dc8326e8..0000000000 --- a/interface/src/devices/DdeFaceTracker.cpp +++ /dev/null @@ -1,686 +0,0 @@ -// -// DdeFaceTracker.cpp -// -// -// Created by Clement on 8/2/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "DdeFaceTracker.h" - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "Application.h" -#include "InterfaceLogging.h" -#include "Menu.h" - - -static const QHostAddress DDE_SERVER_ADDR("127.0.0.1"); -static const quint16 DDE_SERVER_PORT = 64204; -static const quint16 DDE_CONTROL_PORT = 64205; -#if defined(Q_OS_WIN) -static const QString DDE_PROGRAM_PATH = "/dde/dde.exe"; -#elif defined(Q_OS_MAC) -static const QString DDE_PROGRAM_PATH = "/dde.app/Contents/MacOS/dde"; -#endif -static const QStringList DDE_ARGUMENTS = QStringList() - << "--udp=" + DDE_SERVER_ADDR.toString() + ":" + QString::number(DDE_SERVER_PORT) - << "--receiver=" + QString::number(DDE_CONTROL_PORT) - << "--facedet_interval=500" // ms - << "--headless"; - -static const int NUM_EXPRESSIONS = 46; -static const int MIN_PACKET_SIZE = (8 + NUM_EXPRESSIONS) * sizeof(float) + sizeof(int); -static const int MAX_NAME_SIZE = 31; - -// There's almost but not quite a 1-1 correspondence between DDE's 46 and Faceshift 1.3's 48 packets. -// The best guess at mapping is to: -// - Swap L and R values -// - Skip two Faceshift values: JawChew (22) and LipsLowerDown (37) -static const int DDE_TO_FACESHIFT_MAPPING[] = { - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, - 16, - 18, 17, - 19, - 23, - 21, - // Skip JawChew - 20, - 25, 24, 27, 26, 29, 28, 31, 30, 33, 32, - 34, 35, 36, - // Skip LipsLowerDown - 38, 39, 40, 41, 42, 43, 44, 45, - 47, 46 -}; - -// The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much -// less than this. -static const float DDE_COEFFICIENT_SCALES[] = { - 1.0f, // EyeBlink_L - 1.0f, // EyeBlink_R - 1.0f, // EyeSquint_L - 1.0f, // EyeSquint_R - 1.0f, // EyeDown_L - 1.0f, // EyeDown_R - 1.0f, // EyeIn_L - 1.0f, // EyeIn_R - 1.0f, // EyeOpen_L - 1.0f, // EyeOpen_R - 1.0f, // EyeOut_L - 1.0f, // EyeOut_R - 1.0f, // EyeUp_L - 1.0f, // EyeUp_R - 3.0f, // BrowsD_L - 3.0f, // BrowsD_R - 3.0f, // BrowsU_C - 3.0f, // BrowsU_L - 3.0f, // BrowsU_R - 1.0f, // JawFwd - 2.0f, // JawLeft - 1.8f, // JawOpen - 1.0f, // JawChew - 2.0f, // JawRight - 1.5f, // MouthLeft - 1.5f, // MouthRight - 1.5f, // MouthFrown_L - 1.5f, // MouthFrown_R - 2.5f, // MouthSmile_L - 2.5f, // MouthSmile_R - 1.0f, // MouthDimple_L - 1.0f, // MouthDimple_R - 1.0f, // LipsStretch_L - 1.0f, // LipsStretch_R - 1.0f, // LipsUpperClose - 1.0f, // LipsLowerClose - 1.0f, // LipsUpperUp - 1.0f, // LipsLowerDown - 1.0f, // LipsUpperOpen - 1.0f, // LipsLowerOpen - 1.5f, // LipsFunnel - 2.5f, // LipsPucker - 1.5f, // ChinLowerRaise - 1.5f, // ChinUpperRaise - 1.0f, // Sneer - 3.0f, // Puff - 1.0f, // CheekSquint_L - 1.0f // CheekSquint_R -}; - -struct DDEPacket { - //roughly in mm - float focal_length[1]; - float translation[3]; - - //quaternion - float rotation[4]; - - // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much - // less than this. - float expressions[NUM_EXPRESSIONS]; - - //avatar id selected on the UI - int avatar_id; - - //client name, arbitrary length - char name[MAX_NAME_SIZE + 1]; -}; - -static const float STARTING_DDE_MESSAGE_TIME = 0.033f; -static const float DEFAULT_DDE_EYE_CLOSING_THRESHOLD = 0.8f; -static const int CALIBRATION_SAMPLES = 150; - -DdeFaceTracker::DdeFaceTracker() : - DdeFaceTracker(QHostAddress::Any, DDE_SERVER_PORT, DDE_CONTROL_PORT) -{ - -} - -DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, quint16 controlPort) : - _ddeProcess(NULL), - _ddeStopping(false), - _host(host), - _serverPort(serverPort), - _controlPort(controlPort), - _lastReceiveTimestamp(0), - _reset(false), - _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes - _rightBlinkIndex(1), - _leftEyeDownIndex(4), - _rightEyeDownIndex(5), - _leftEyeInIndex(6), - _rightEyeInIndex(7), - _leftEyeOpenIndex(8), - _rightEyeOpenIndex(9), - _browDownLeftIndex(14), - _browDownRightIndex(15), - _browUpCenterIndex(16), - _browUpLeftIndex(17), - _browUpRightIndex(18), - _mouthSmileLeftIndex(28), - _mouthSmileRightIndex(29), - _jawOpenIndex(21), - _lastMessageReceived(0), - _averageMessageTime(STARTING_DDE_MESSAGE_TIME), - _lastHeadTranslation(glm::vec3(0.0f)), - _filteredHeadTranslation(glm::vec3(0.0f)), - _lastBrowUp(0.0f), - _filteredBrowUp(0.0f), - _eyePitch(0.0f), - _eyeYaw(0.0f), - _lastEyePitch(0.0f), - _lastEyeYaw(0.0f), - _filteredEyePitch(0.0f), - _filteredEyeYaw(0.0f), - _longTermAverageEyePitch(0.0f), - _longTermAverageEyeYaw(0.0f), - _lastEyeBlinks(), - _filteredEyeBlinks(), - _lastEyeCoefficients(), - _eyeClosingThreshold("ddeEyeClosingThreshold", DEFAULT_DDE_EYE_CLOSING_THRESHOLD), - _isCalibrating(false), - _calibrationCount(0), - _calibrationValues(), - _calibrationBillboard(NULL), - _calibrationMessage(QString()), - _isCalibrated(false) -{ - _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES); - _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES); - _coefficientAverages.resize(NUM_FACESHIFT_BLENDSHAPES); - _calibrationValues.resize(NUM_FACESHIFT_BLENDSHAPES); - - _eyeStates[0] = EYE_UNCONTROLLED; - _eyeStates[1] = EYE_UNCONTROLLED; - - connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); - connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError))); - connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), - SLOT(socketStateChanged(QAbstractSocket::SocketState))); -} - -DdeFaceTracker::~DdeFaceTracker() { - setEnabled(false); - - if (_isCalibrating) { - cancelCalibration(); - } -} - -void DdeFaceTracker::init() { - FaceTracker::init(); - setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::UseCamera) && !_isMuted); - Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setEnabled(!_isMuted); -} - -void DdeFaceTracker::setEnabled(bool enabled) { - if (!_isInitialized) { - // Don't enable until have explicitly initialized - return; - } -#ifdef HAVE_DDE - - if (_isCalibrating) { - cancelCalibration(); - } - - // isOpen() does not work as one might expect on QUdpSocket; don't test isOpen() before closing socket. - _udpSocket.close(); - - // Terminate any existing DDE process, perhaps left running after an Interface crash. - // Do this even if !enabled in case user reset their settings after crash. - const char* DDE_EXIT_COMMAND = "exit"; - _udpSocket.bind(_host, _serverPort); - _udpSocket.writeDatagram(DDE_EXIT_COMMAND, DDE_SERVER_ADDR, _controlPort); - - if (enabled && !_ddeProcess) { - _ddeStopping = false; - qCDebug(interfaceapp) << "DDE Face Tracker: Starting"; - _ddeProcess = new QProcess(qApp); - connect(_ddeProcess, SIGNAL(finished(int, QProcess::ExitStatus)), SLOT(processFinished(int, QProcess::ExitStatus))); - _ddeProcess->start(QCoreApplication::applicationDirPath() + DDE_PROGRAM_PATH, DDE_ARGUMENTS); - } - - if (!enabled && _ddeProcess) { - _ddeStopping = true; - qCDebug(interfaceapp) << "DDE Face Tracker: Stopping"; - } -#endif -} - -void DdeFaceTracker::processFinished(int exitCode, QProcess::ExitStatus exitStatus) { - if (_ddeProcess) { - if (_ddeStopping) { - qCDebug(interfaceapp) << "DDE Face Tracker: Stopped"; - - } else { - qCWarning(interfaceapp) << "DDE Face Tracker: Stopped unexpectedly"; - Menu::getInstance()->setIsOptionChecked(MenuOption::NoFaceTracking, true); - } - _udpSocket.close(); - delete _ddeProcess; - _ddeProcess = NULL; - } -} - -void DdeFaceTracker::reset() { - if (_udpSocket.state() == QAbstractSocket::BoundState) { - _reset = true; - - qCDebug(interfaceapp) << "DDE Face Tracker: Reset"; - - const char* DDE_RESET_COMMAND = "reset"; - _udpSocket.writeDatagram(DDE_RESET_COMMAND, DDE_SERVER_ADDR, _controlPort); - - FaceTracker::reset(); - - _reset = true; - } -} - -void DdeFaceTracker::update(float deltaTime) { - if (!isActive()) { - return; - } - FaceTracker::update(deltaTime); - - glm::vec3 headEulers = glm::degrees(glm::eulerAngles(_headRotation)); - _estimatedEyePitch = _eyePitch - headEulers.x; - _estimatedEyeYaw = _eyeYaw - headEulers.y; -} - -bool DdeFaceTracker::isActive() const { - return (_ddeProcess != NULL); -} - -bool DdeFaceTracker::isTracking() const { - static const quint64 ACTIVE_TIMEOUT_USECS = 3000000; //3 secs - return (usecTimestampNow() - _lastReceiveTimestamp < ACTIVE_TIMEOUT_USECS); -} - -//private slots and methods -void DdeFaceTracker::socketErrorOccurred(QAbstractSocket::SocketError socketError) { - qCWarning(interfaceapp) << "DDE Face Tracker: Socket error: " << _udpSocket.errorString(); -} - -void DdeFaceTracker::socketStateChanged(QAbstractSocket::SocketState socketState) { - QString state; - switch(socketState) { - case QAbstractSocket::BoundState: - state = "Bound"; - break; - case QAbstractSocket::ClosingState: - state = "Closing"; - break; - case QAbstractSocket::ConnectedState: - state = "Connected"; - break; - case QAbstractSocket::ConnectingState: - state = "Connecting"; - break; - case QAbstractSocket::HostLookupState: - state = "Host Lookup"; - break; - case QAbstractSocket::ListeningState: - state = "Listening"; - break; - case QAbstractSocket::UnconnectedState: - state = "Unconnected"; - break; - } - qCDebug(interfaceapp) << "DDE Face Tracker: Socket: " << state; -} - -void DdeFaceTracker::readPendingDatagrams() { - QByteArray buffer; - while (_udpSocket.hasPendingDatagrams()) { - buffer.resize(_udpSocket.pendingDatagramSize()); - _udpSocket.readDatagram(buffer.data(), buffer.size()); - } - decodePacket(buffer); -} - -float DdeFaceTracker::getBlendshapeCoefficient(int index) const { - return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; -} - -void DdeFaceTracker::decodePacket(const QByteArray& buffer) { - _lastReceiveTimestamp = usecTimestampNow(); - - if (buffer.size() > MIN_PACKET_SIZE) { - if (!_isCalibrated) { - calibrate(); - } - - bool isFiltering = Menu::getInstance()->isOptionChecked(MenuOption::VelocityFilter); - - DDEPacket packet; - int bytesToCopy = glm::min((int)sizeof(packet), buffer.size()); - memset(&packet.name, '\n', MAX_NAME_SIZE + 1); - memcpy(&packet, buffer.data(), bytesToCopy); - - glm::vec3 translation; - memcpy(&translation, packet.translation, sizeof(packet.translation)); - glm::quat rotation; - memcpy(&rotation, &packet.rotation, sizeof(packet.rotation)); - if (_reset || (_lastMessageReceived == 0)) { - memcpy(&_referenceTranslation, &translation, sizeof(glm::vec3)); - memcpy(&_referenceRotation, &rotation, sizeof(glm::quat)); - _reset = false; - } - - // Compute relative translation - float LEAN_DAMPING_FACTOR = 75.0f; - translation -= _referenceTranslation; - translation /= LEAN_DAMPING_FACTOR; - translation.x *= -1; - if (isFiltering) { - glm::vec3 linearVelocity = (translation - _lastHeadTranslation) / _averageMessageTime; - const float LINEAR_VELOCITY_FILTER_STRENGTH = 0.3f; - float velocityFilter = glm::clamp(1.0f - glm::length(linearVelocity) * - LINEAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredHeadTranslation = velocityFilter * _filteredHeadTranslation + (1.0f - velocityFilter) * translation; - _lastHeadTranslation = translation; - _headTranslation = _filteredHeadTranslation; - } else { - _headTranslation = translation; - } - - // Compute relative rotation - rotation = glm::inverse(_referenceRotation) * rotation; - if (isFiltering) { - glm::quat r = glm::normalize(rotation * glm::inverse(_headRotation)); - float theta = 2 * acos(r.w); - glm::vec3 angularVelocity; - if (theta > EPSILON) { - float rMag = glm::length(glm::vec3(r.x, r.y, r.z)); - angularVelocity = theta / _averageMessageTime * glm::vec3(r.x, r.y, r.z) / rMag; - } else { - angularVelocity = glm::vec3(0, 0, 0); - } - const float ANGULAR_VELOCITY_FILTER_STRENGTH = 0.3f; - _headRotation = safeMix(_headRotation, rotation, glm::clamp(glm::length(angularVelocity) * - ANGULAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f)); - } else { - _headRotation = rotation; - } - - // Translate DDE coefficients to Faceshift compatible coefficients - for (int i = 0; i < NUM_EXPRESSIONS; i++) { - _coefficients[DDE_TO_FACESHIFT_MAPPING[i]] = packet.expressions[i]; - } - - // Calibration - if (_isCalibrating) { - addCalibrationDatum(); - } - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _coefficients[i] -= _coefficientAverages[i]; - } - - // Use BrowsU_C to control both brows' up and down - float browUp = _coefficients[_browUpCenterIndex]; - if (isFiltering) { - const float BROW_VELOCITY_FILTER_STRENGTH = 0.5f; - float velocity = fabsf(browUp - _lastBrowUp) / _averageMessageTime; - float velocityFilter = glm::clamp(velocity * BROW_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredBrowUp = velocityFilter * browUp + (1.0f - velocityFilter) * _filteredBrowUp; - _lastBrowUp = browUp; - browUp = _filteredBrowUp; - _coefficients[_browUpCenterIndex] = browUp; - } - _coefficients[_browUpLeftIndex] = browUp; - _coefficients[_browUpRightIndex] = browUp; - _coefficients[_browDownLeftIndex] = -browUp; - _coefficients[_browDownRightIndex] = -browUp; - - // Offset jaw open coefficient - static const float JAW_OPEN_THRESHOLD = 0.1f; - _coefficients[_jawOpenIndex] = _coefficients[_jawOpenIndex] - JAW_OPEN_THRESHOLD; - - // Offset smile coefficients - static const float SMILE_THRESHOLD = 0.5f; - _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD; - _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD; - - // Eye pitch and yaw - // EyeDown coefficients work better over both +ve and -ve values than EyeUp values. - // EyeIn coefficients work better over both +ve and -ve values than EyeOut values. - // Pitch and yaw values are relative to the screen. - const float EYE_PITCH_SCALE = -1500.0f; // Sign, scale, and average to be similar to Faceshift values. - _eyePitch = EYE_PITCH_SCALE * (_coefficients[_leftEyeDownIndex] + _coefficients[_rightEyeDownIndex]); - const float EYE_YAW_SCALE = 2000.0f; // Scale and average to be similar to Faceshift values. - _eyeYaw = EYE_YAW_SCALE * (_coefficients[_leftEyeInIndex] + _coefficients[_rightEyeInIndex]); - if (isFiltering) { - const float EYE_VELOCITY_FILTER_STRENGTH = 0.005f; - float pitchVelocity = fabsf(_eyePitch - _lastEyePitch) / _averageMessageTime; - float pitchVelocityFilter = glm::clamp(pitchVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredEyePitch = pitchVelocityFilter * _eyePitch + (1.0f - pitchVelocityFilter) * _filteredEyePitch; - _lastEyePitch = _eyePitch; - _eyePitch = _filteredEyePitch; - float yawVelocity = fabsf(_eyeYaw - _lastEyeYaw) / _averageMessageTime; - float yawVelocityFilter = glm::clamp(yawVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredEyeYaw = yawVelocityFilter * _eyeYaw + (1.0f - yawVelocityFilter) * _filteredEyeYaw; - _lastEyeYaw = _eyeYaw; - _eyeYaw = _filteredEyeYaw; - } - - // Velocity filter EyeBlink values - const float DDE_EYEBLINK_SCALE = 3.0f; - float eyeBlinks[] = { DDE_EYEBLINK_SCALE * _coefficients[_leftBlinkIndex], - DDE_EYEBLINK_SCALE * _coefficients[_rightBlinkIndex] }; - if (isFiltering) { - const float BLINK_VELOCITY_FILTER_STRENGTH = 0.3f; - for (int i = 0; i < 2; i++) { - float velocity = fabsf(eyeBlinks[i] - _lastEyeBlinks[i]) / _averageMessageTime; - float velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredEyeBlinks[i] = velocityFilter * eyeBlinks[i] + (1.0f - velocityFilter) * _filteredEyeBlinks[i]; - _lastEyeBlinks[i] = eyeBlinks[i]; - } - } - - // Finesse EyeBlink values - float eyeCoefficients[2]; - if (Menu::getInstance()->isOptionChecked(MenuOption::BinaryEyelidControl)) { - if (_eyeStates[0] == EYE_UNCONTROLLED) { - _eyeStates[0] = EYE_OPEN; - _eyeStates[1] = EYE_OPEN; - } - - for (int i = 0; i < 2; i++) { - // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen - // -ve values control EyeOpen; +ve values control EyeBlink - static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value - eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD); - - // Change to closing or opening states - const float EYE_CONTROL_HYSTERISIS = 0.25f; - float eyeClosingThreshold = getEyeClosingThreshold(); - float eyeOpeningThreshold = eyeClosingThreshold - EYE_CONTROL_HYSTERISIS; - if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > eyeClosingThreshold) { - _eyeStates[i] = EYE_CLOSING; - } else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING) - && eyeCoefficients[i] < eyeOpeningThreshold) { - _eyeStates[i] = EYE_OPENING; - } - - const float EYELID_MOVEMENT_RATE = 10.0f; // units/second - const float EYE_OPEN_SCALE = 0.2f; - if (_eyeStates[i] == EYE_CLOSING) { - // Close eyelid until it's fully closed - float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime; - if (closingValue >= 1.0f) { - _eyeStates[i] = EYE_CLOSED; - eyeCoefficients[i] = 1.0f; - } else { - eyeCoefficients[i] = closingValue; - } - } else if (_eyeStates[i] == EYE_OPENING) { - // Open eyelid until it meets the current adjusted value - float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime; - if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) { - _eyeStates[i] = EYE_OPEN; - eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else { - eyeCoefficients[i] = openingValue; - } - } else if (_eyeStates[i] == EYE_OPEN) { - // Reduce eyelid movement - eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else if (_eyeStates[i] == EYE_CLOSED) { - // Keep eyelid fully closed - eyeCoefficients[i] = 1.0; - } - } - - if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) { - // Couple eyelids - eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f; - } - - _lastEyeCoefficients[0] = eyeCoefficients[0]; - _lastEyeCoefficients[1] = eyeCoefficients[1]; - } else { - _eyeStates[0] = EYE_UNCONTROLLED; - _eyeStates[1] = EYE_UNCONTROLLED; - - eyeCoefficients[0] = _filteredEyeBlinks[0]; - eyeCoefficients[1] = _filteredEyeBlinks[1]; - } - - // Couple eyelid values if configured - use the most "open" value for both - if (Menu::getInstance()->isOptionChecked(MenuOption::CoupleEyelids)) { - float eyeCoefficient = std::min(eyeCoefficients[0], eyeCoefficients[1]); - eyeCoefficients[0] = eyeCoefficient; - eyeCoefficients[1] = eyeCoefficient; - } - - // Use EyeBlink values to control both EyeBlink and EyeOpen - if (eyeCoefficients[0] > 0) { - _coefficients[_leftBlinkIndex] = eyeCoefficients[0]; - _coefficients[_leftEyeOpenIndex] = 0.0f; - } else { - _coefficients[_leftBlinkIndex] = 0.0f; - _coefficients[_leftEyeOpenIndex] = -eyeCoefficients[0]; - } - if (eyeCoefficients[1] > 0) { - _coefficients[_rightBlinkIndex] = eyeCoefficients[1]; - _coefficients[_rightEyeOpenIndex] = 0.0f; - } else { - _coefficients[_rightBlinkIndex] = 0.0f; - _coefficients[_rightEyeOpenIndex] = -eyeCoefficients[1]; - } - - // Scale all coefficients - for (int i = 0; i < NUM_EXPRESSIONS; i++) { - _blendshapeCoefficients[i] - = glm::clamp(DDE_COEFFICIENT_SCALES[i] * _coefficients[i], 0.0f, 1.0f); - } - - // Calculate average frame time - const float FRAME_AVERAGING_FACTOR = 0.99f; - quint64 usecsNow = usecTimestampNow(); - if (_lastMessageReceived != 0) { - _averageMessageTime = FRAME_AVERAGING_FACTOR * _averageMessageTime - + (1.0f - FRAME_AVERAGING_FACTOR) * (float)(usecsNow - _lastMessageReceived) / 1000000.0f; - } - _lastMessageReceived = usecsNow; - - FaceTracker::countFrame(); - - } else { - qCWarning(interfaceapp) << "DDE Face Tracker: Decode error"; - } - - if (_isCalibrating && _calibrationCount > CALIBRATION_SAMPLES) { - finishCalibration(); - } -} - -void DdeFaceTracker::setEyeClosingThreshold(float eyeClosingThreshold) { - _eyeClosingThreshold.set(eyeClosingThreshold); -} - -static const int CALIBRATION_BILLBOARD_WIDTH = 300; -static const int CALIBRATION_BILLBOARD_HEIGHT = 120; -static QString CALIBRATION_INSTRUCTION_MESSAGE = "Hold still to calibrate camera"; - -void DdeFaceTracker::calibrate() { - if (!Menu::getInstance()->isOptionChecked(MenuOption::UseCamera) || _isMuted) { - return; - } - - if (!_isCalibrating) { - qCDebug(interfaceapp) << "DDE Face Tracker: Calibration started"; - - _isCalibrating = true; - _calibrationCount = 0; - _calibrationMessage = CALIBRATION_INSTRUCTION_MESSAGE + "\n\n"; - - // FIXME: this overlay probably doesn't work anymore - _calibrationBillboard = new TextOverlay(); - glm::vec2 viewport = qApp->getCanvasSize(); - _calibrationBillboard->setX((viewport.x - CALIBRATION_BILLBOARD_WIDTH) / 2); - _calibrationBillboard->setY((viewport.y - CALIBRATION_BILLBOARD_HEIGHT) / 2); - _calibrationBillboard->setWidth(CALIBRATION_BILLBOARD_WIDTH); - _calibrationBillboard->setHeight(CALIBRATION_BILLBOARD_HEIGHT); - _calibrationBillboardID = qApp->getOverlays().addOverlay(_calibrationBillboard); - - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _calibrationValues[i] = 0.0f; - } - } -} - -void DdeFaceTracker::addCalibrationDatum() { - const int LARGE_TICK_INTERVAL = 30; - const int SMALL_TICK_INTERVAL = 6; - int samplesLeft = CALIBRATION_SAMPLES - _calibrationCount; - if (samplesLeft % LARGE_TICK_INTERVAL == 0) { - _calibrationMessage += QString::number(samplesLeft / LARGE_TICK_INTERVAL); - // FIXME: set overlay text - } else if (samplesLeft % SMALL_TICK_INTERVAL == 0) { - _calibrationMessage += "."; - // FIXME: set overlay text - } - - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _calibrationValues[i] += _coefficients[i]; - } - - _calibrationCount += 1; -} - -void DdeFaceTracker::cancelCalibration() { - qApp->getOverlays().deleteOverlay(_calibrationBillboardID); - _calibrationBillboard = NULL; - _isCalibrating = false; - qCDebug(interfaceapp) << "DDE Face Tracker: Calibration cancelled"; -} - -void DdeFaceTracker::finishCalibration() { - qApp->getOverlays().deleteOverlay(_calibrationBillboardID); - _calibrationBillboard = NULL; - _isCalibrating = false; - _isCalibrated = true; - - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _coefficientAverages[i] = _calibrationValues[i] / (float)CALIBRATION_SAMPLES; - } - - reset(); - - qCDebug(interfaceapp) << "DDE Face Tracker: Calibration finished"; -} diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h deleted file mode 100644 index dc451134f0..0000000000 --- a/interface/src/devices/DdeFaceTracker.h +++ /dev/null @@ -1,181 +0,0 @@ -// -// DdeFaceTracker.h -// -// -// Created by Clement on 8/2/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_DdeFaceTracker_h -#define hifi_DdeFaceTracker_h - -#include - -//Disabling dde due to random crashes with closing the socket on macos. all the accompanying code is wrapped with the ifdef HAVE_DDE. uncomment the define below to enable -#if defined(Q_OS_WIN) || defined(Q_OS_OSX) - //#define HAVE_DDE -#endif - -#include -#include - -#include -#include - -#include - -/**jsdoc - * The FaceTracker API helps manage facial tracking hardware. - * @namespace FaceTracker - * - * @hifi-interface - * @hifi-client-entity - * @hifi-avatar - */ - -class DdeFaceTracker : public FaceTracker, public Dependency { - Q_OBJECT - SINGLETON_DEPENDENCY - -public: - virtual void init() override; - virtual void reset() override; - virtual void update(float deltaTime) override; - - virtual bool isActive() const override; - virtual bool isTracking() const override; - - float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } - float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } - float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } - float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); } - - float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); } - float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); } - float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); } - float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); } - float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); } - - float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); } - float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } - float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } - - float getEyeClosingThreshold() { return _eyeClosingThreshold.get(); } - void setEyeClosingThreshold(float eyeClosingThreshold); - -public slots: - - /**jsdoc - * @function FaceTracker.setEnabled - * @param {boolean} enabled - */ - void setEnabled(bool enabled) override; - - /**jsdoc - * @function FaceTracker.calibrate - */ - void calibrate(); - -private slots: - void processFinished(int exitCode, QProcess::ExitStatus exitStatus); - - //sockets - void socketErrorOccurred(QAbstractSocket::SocketError socketError); - void readPendingDatagrams(); - void socketStateChanged(QAbstractSocket::SocketState socketState); - -private: - DdeFaceTracker(); - DdeFaceTracker(const QHostAddress& host, quint16 serverPort, quint16 controlPort); - virtual ~DdeFaceTracker(); - - QProcess* _ddeProcess; - bool _ddeStopping; - - QHostAddress _host; - quint16 _serverPort; - quint16 _controlPort; - - float getBlendshapeCoefficient(int index) const; - void decodePacket(const QByteArray& buffer); - - // sockets - QUdpSocket _udpSocket; - quint64 _lastReceiveTimestamp; - - bool _reset; - glm::vec3 _referenceTranslation; - glm::quat _referenceRotation; - - int _leftBlinkIndex; - int _rightBlinkIndex; - int _leftEyeDownIndex; - int _rightEyeDownIndex; - int _leftEyeInIndex; - int _rightEyeInIndex; - int _leftEyeOpenIndex; - int _rightEyeOpenIndex; - - int _browDownLeftIndex; - int _browDownRightIndex; - int _browUpCenterIndex; - int _browUpLeftIndex; - int _browUpRightIndex; - - int _mouthSmileLeftIndex; - int _mouthSmileRightIndex; - - int _jawOpenIndex; - - QVector _coefficients; - - quint64 _lastMessageReceived; - float _averageMessageTime; - - glm::vec3 _lastHeadTranslation; - glm::vec3 _filteredHeadTranslation; - - float _lastBrowUp; - float _filteredBrowUp; - - float _eyePitch; // Degrees, relative to screen - float _eyeYaw; - float _lastEyePitch; - float _lastEyeYaw; - float _filteredEyePitch; - float _filteredEyeYaw; - float _longTermAverageEyePitch = 0.0f; - float _longTermAverageEyeYaw = 0.0f; - bool _longTermAverageInitialized = false; - - enum EyeState { - EYE_UNCONTROLLED, - EYE_OPEN, - EYE_CLOSING, - EYE_CLOSED, - EYE_OPENING - }; - EyeState _eyeStates[2]; - float _lastEyeBlinks[2]; - float _filteredEyeBlinks[2]; - float _lastEyeCoefficients[2]; - Setting::Handle _eyeClosingThreshold; - - QVector _coefficientAverages; - - bool _isCalibrating; - int _calibrationCount; - QVector _calibrationValues; - TextOverlay* _calibrationBillboard; - QUuid _calibrationBillboardID; - QString _calibrationMessage; - bool _isCalibrated; - void addCalibrationDatum(); - void cancelCalibration(); - void finishCalibration(); -}; - -#endif // hifi_DdeFaceTracker_h diff --git a/interface/src/ui/AvatarInputs.cpp b/interface/src/ui/AvatarInputs.cpp index a6bc7cf84f..3986cb1533 100644 --- a/interface/src/ui/AvatarInputs.cpp +++ b/interface/src/ui/AvatarInputs.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include "Application.h" @@ -76,8 +75,6 @@ void AvatarInputs::update() { return; } - AI_UPDATE(cameraEnabled, !Menu::getInstance()->isOptionChecked(MenuOption::NoFaceTracking)); - AI_UPDATE(cameraMuted, Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking)); AI_UPDATE(isHMD, qApp->isHMDMode()); } @@ -103,13 +100,6 @@ bool AvatarInputs::getIgnoreRadiusEnabled() const { return DependencyManager::get()->getIgnoreRadiusEnabled(); } -void AvatarInputs::toggleCameraMute() { - FaceTracker* faceTracker = qApp->getSelectedFaceTracker(); - if (faceTracker) { - faceTracker->toggleMute(); - } -} - void AvatarInputs::resetSensors() { qApp->resetSensors(); } diff --git a/interface/src/ui/AvatarInputs.h b/interface/src/ui/AvatarInputs.h index dca39fd433..40ee0ea7d4 100644 --- a/interface/src/ui/AvatarInputs.h +++ b/interface/src/ui/AvatarInputs.h @@ -32,14 +32,6 @@ class AvatarInputs : public QObject { * @hifi-client-entity * @hifi-avatar * - * @property {boolean} cameraEnabled - true if webcam face tracking is enabled, false if it is - * disabled. - * Read-only. - *

Deprecated: This property is deprecated and will be removed.

- * @property {boolean} cameraMuted - true if webcam face tracking is muted (temporarily disabled), - * false it if isn't. - * Read-only. - *

Deprecated: This property is deprecated and will be removed.

* @property {boolean} ignoreRadiusEnabled - true if the privacy shield is enabled, false if it * is disabled. * Read-only. @@ -51,8 +43,6 @@ class AvatarInputs : public QObject { * it is hidden. */ - AI_PROPERTY(bool, cameraEnabled, false) - AI_PROPERTY(bool, cameraMuted, false) AI_PROPERTY(bool, isHMD, false) Q_PROPERTY(bool showAudioTools READ showAudioTools WRITE setShowAudioTools NOTIFY showAudioToolsChanged) @@ -97,22 +87,6 @@ public slots: signals: - /**jsdoc - * Triggered when webcam face tracking is enabled or disabled. - * @deprecated This signal is deprecated and will be removed. - * @function AvatarInputs.cameraEnabledChanged - * @returns {Signal} - */ - void cameraEnabledChanged(); - - /**jsdoc - * Triggered when webcam face tracking is muted (temporarily disabled) or unmuted. - * @deprecated This signal is deprecated and will be removed. - * @function AvatarInputs.cameraMutedChanged - * @returns {Signal} - */ - void cameraMutedChanged(); - /**jsdoc * Triggered when the display mode changes between desktop and HMD. * @function AvatarInputs.isHMDChanged @@ -183,13 +157,6 @@ protected: */ Q_INVOKABLE void resetSensors(); - /**jsdoc - * Toggles the muting (temporary disablement) of webcam face tracking on/off. - *

Deprecated: This function is deprecated and will be removed.

- * @function AvatarInputs.toggleCameraMute - */ - Q_INVOKABLE void toggleCameraMute(); - private: void onAvatarEnteredIgnoreRadius(); void onAvatarLeftIgnoreRadius(); diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index ec15dd8111..5f033fe008 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -10,7 +10,6 @@ #include #include -#include #include #include #include @@ -285,22 +284,6 @@ void setupPreferences() { preferences->addPreference(preference); } - static const QString FACE_TRACKING{ "Face Tracking" }; - { -#ifdef HAVE_DDE - auto getter = []()->float { return DependencyManager::get()->getEyeClosingThreshold(); }; - auto setter = [](float value) { DependencyManager::get()->setEyeClosingThreshold(value); }; - preferences->addPreference(new SliderPreference(FACE_TRACKING, "Eye Closing Threshold", getter, setter)); -#endif - } - - - { - auto getter = []()->float { return FaceTracker::getEyeDeflection(); }; - auto setter = [](float value) { FaceTracker::setEyeDeflection(value); }; - preferences->addPreference(new SliderPreference(FACE_TRACKING, "Eye Deflection", getter, setter)); - } - static const QString VR_MOVEMENT{ "VR Movement" }; { auto getter = [myAvatar]()->bool { return myAvatar->getAllowTeleporting(); }; diff --git a/libraries/avatars-renderer/CMakeLists.txt b/libraries/avatars-renderer/CMakeLists.txt index de1ac1a7c2..7929be8b04 100644 --- a/libraries/avatars-renderer/CMakeLists.txt +++ b/libraries/avatars-renderer/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET_NAME avatars-renderer) setup_hifi_library(Network Script) -link_hifi_libraries(shared shaders gpu graphics animation material-networking model-networking script-engine render render-utils image trackers entities-renderer) +link_hifi_libraries(shared shaders gpu graphics animation material-networking model-networking script-engine render render-utils image entities-renderer) include_hifi_library_headers(avatars) include_hifi_library_headers(networking) include_hifi_library_headers(hfm) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 63d8e2981c..ea6cdd7087 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include "Logging.h" @@ -26,6 +25,22 @@ using namespace std; static bool disableEyelidAdjustment { false }; +static void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, + float jawOpen, float mouth2, float mouth3, float mouth4, QVector& coefficients) { + + coefficients.resize(std::max((int)coefficients.size(), (int)Blendshapes::BlendshapeCount)); + qFill(coefficients.begin(), coefficients.end(), 0.0f); + coefficients[(int)Blendshapes::EyeBlink_L] = leftBlink; + coefficients[(int)Blendshapes::EyeBlink_R] = rightBlink; + coefficients[(int)Blendshapes::BrowsU_C] = browUp; + coefficients[(int)Blendshapes::BrowsU_L] = browUp; + coefficients[(int)Blendshapes::BrowsU_R] = browUp; + coefficients[(int)Blendshapes::JawOpen] = jawOpen; + coefficients[(int)Blendshapes::MouthSmile_L] = coefficients[(int)Blendshapes::MouthSmile_R] = mouth4; + coefficients[(int)Blendshapes::LipsUpperClose] = mouth2; + coefficients[(int)Blendshapes::LipsFunnel] = mouth3; +} + Head::Head(Avatar* owningAvatar) : HeadData(owningAvatar), _leftEyeLookAtID(DependencyManager::get()->allocateID()), @@ -153,7 +168,7 @@ void Head::simulate(float deltaTime) { _mouthTime = 0.0f; } - FaceTracker::updateFakeCoefficients( + updateFakeCoefficients( _leftEyeBlink, _rightEyeBlink, _browAudioLift, diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 0e0dd9887f..561f2c798a 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -71,7 +71,7 @@ void HeadData::setOrientation(const glm::quat& orientation) { } void HeadData::computeBlendshapesLookupMap(){ - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { _blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i; } } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index e5dd158505..c6bcc2599b 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -20,7 +20,7 @@ #include #include -#include +#include // degrees const float MIN_HEAD_YAW = -180.0f; diff --git a/libraries/fbx/src/FBXSerializer.cpp b/libraries/fbx/src/FBXSerializer.cpp index f8339ddd31..100f6ee98e 100644 --- a/libraries/fbx/src/FBXSerializer.cpp +++ b/libraries/fbx/src/FBXSerializer.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp index da21c7995e..5d4daf53f7 100755 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include "FBXSerializer.h" diff --git a/libraries/shared/src/FaceshiftConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp similarity index 84% rename from libraries/shared/src/FaceshiftConstants.cpp rename to libraries/shared/src/BlendshapeConstants.cpp index b847837a7a..528b941b73 100644 --- a/libraries/shared/src/FaceshiftConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -1,5 +1,5 @@ // -// FaceshiftConstants.cpp +// BlendshapeConstants.cpp // // // Created by Clement on 1/23/15. @@ -9,7 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include "FaceshiftConstants.h" +#include "BlendshapeConstants.h" const char* FACESHIFT_BLENDSHAPES[] = { "EyeBlink_L", @@ -34,7 +34,7 @@ const char* FACESHIFT_BLENDSHAPES[] = { "JawFwd", "JawLeft", "JawOpen", - "JawChew", // legacy not in ARKit + "JawChew", "JawRight", "MouthLeft", "MouthRight", @@ -48,15 +48,15 @@ const char* FACESHIFT_BLENDSHAPES[] = { "LipsStretch_R", "LipsUpperClose", "LipsLowerClose", - "LipsUpperUp", // legacy, split in ARKit - "LipsLowerDown", // legacy, split in ARKit + "LipsUpperUp", + "LipsLowerDown", "LipsUpperOpen", "LipsLowerOpen", "LipsFunnel", "LipsPucker", "ChinLowerRaise", "ChinUpperRaise", - "Sneer", // legacy, split in ARKit + "Sneer", "Puff", "CheekSquint_L", "CheekSquint_R", @@ -71,8 +71,6 @@ const char* FACESHIFT_BLENDSHAPES[] = { // MouthShrugUpper // TongueOut -const int NUM_FACESHIFT_BLENDSHAPES = sizeof(FACESHIFT_BLENDSHAPES) / sizeof(char*); - const int EYE_BLINK_L_INDEX = 0; const int EYE_BLINK_R_INDEX = 1; const int EYE_SQUINT_L_INDEX = 2; diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h new file mode 100644 index 0000000000..0fca03e34b --- /dev/null +++ b/libraries/shared/src/BlendshapeConstants.h @@ -0,0 +1,76 @@ +// +// BlendshapeConstants.h +// +// +// Created by Clement on 1/23/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_BlendshapeConstants_h +#define hifi_BlendshapeConstants_h + +/// The names of the blendshapes expected by Faceshift, terminated with an empty string. +extern const char* FACESHIFT_BLENDSHAPES[]; + +// Eyes and Brows indices +extern const int EYE_BLINK_INDICES[]; +extern const int EYE_OPEN_INDICES[]; +extern const int BROWS_U_INDICES[]; +extern const int EYE_SQUINT_INDICES[]; + +enum class Blendshapes : int { + EyeBlink_L = 0, + EyeBlink_R, + EyeSquint_L, + EyeSquint_R, + EyeDown_L, + EyeDown_R, + EyeIn_L, + EyeIn_R, + EyeOpen_L, + EyeOpen_R, + EyeOut_L, // 10 + EyeOut_R, + EyeUp_L, + EyeUp_R, + BrowsD_L, + BrowsD_R, + BrowsU_C, + BrowsU_L, + BrowsU_R, + JawFwd, + JawLeft, // 20 + JawOpen, + JawChew, // legacy not in ARKit + JawRight, + MouthLeft, + MouthRight, + MouthFrown_L, + MouthFrown_R, + MouthSmile_L, + MouthSmile_R, + MouthDimple_L, // 30 + MouthDimple_R, + LipsStretch_L, + LipsStretch_R, + LipsUpperClose, + LipsLowerClose, + LipsUpperUp, // legacy, split in ARKit + LipsLowerDown, // legacy, split in ARKit + LipsUpperOpen, + LipsLowerOpen, + LipsFunnel, // 40 + LipsPucker, + ChinLowerRaise, + ChinUpperRaise, + Sneer, // legacy, split in ARKit + Puff, + CheekSquint_L, + CheekSquint_R, + BlendshapeCount +}; + +#endif // hifi_BlendshapeConstants_h diff --git a/libraries/shared/src/FaceshiftConstants.h b/libraries/shared/src/FaceshiftConstants.h deleted file mode 100644 index 4349a3a21e..0000000000 --- a/libraries/shared/src/FaceshiftConstants.h +++ /dev/null @@ -1,25 +0,0 @@ -// -// FaceshiftConstants.h -// -// -// Created by Clement on 1/23/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_FaceshiftConstants_h -#define hifi_FaceshiftConstants_h - -/// The names of the blendshapes expected by Faceshift, terminated with an empty string. -extern const char* FACESHIFT_BLENDSHAPES[]; -/// The size of FACESHIFT_BLENDSHAPES -extern const int NUM_FACESHIFT_BLENDSHAPES; -// Eyes and Brows indices -extern const int EYE_BLINK_INDICES[]; -extern const int EYE_OPEN_INDICES[]; -extern const int BROWS_U_INDICES[]; -extern const int EYE_SQUINT_INDICES[]; - -#endif // hifi_FaceshiftConstants_h diff --git a/libraries/trackers/CMakeLists.txt b/libraries/trackers/CMakeLists.txt deleted file mode 100644 index 6943f1a197..0000000000 --- a/libraries/trackers/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(TARGET_NAME trackers) -setup_hifi_library() -GroupSources("src") -link_hifi_libraries(shared) -include_hifi_library_headers(octree) - -target_bullet() diff --git a/libraries/trackers/src/trackers/FaceTracker.cpp b/libraries/trackers/src/trackers/FaceTracker.cpp deleted file mode 100644 index 034787f19a..0000000000 --- a/libraries/trackers/src/trackers/FaceTracker.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// -// Created by Andrzej Kapolka on 4/9/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "FaceTracker.h" - -#include -#include -#include "Logging.h" -//#include "Menu.h" - -const int FPS_TIMER_DELAY = 2000; // ms -const int FPS_TIMER_DURATION = 2000; // ms - -const float DEFAULT_EYE_DEFLECTION = 0.25f; -Setting::Handle FaceTracker::_eyeDeflection("faceshiftEyeDeflection", DEFAULT_EYE_DEFLECTION); -bool FaceTracker::_isMuted { true }; - -void FaceTracker::init() { - _isInitialized = true; // FaceTracker can be used now -} - -inline float FaceTracker::getBlendshapeCoefficient(int index) const { - return isValidBlendshapeIndex(index) ? glm::mix(0.0f, _blendshapeCoefficients[index], getFadeCoefficient()) - : 0.0f; -} - -const QVector& FaceTracker::getBlendshapeCoefficients() const { - static QVector blendshapes; - float fadeCoefficient = getFadeCoefficient(); - if (fadeCoefficient == 1.0f) { - return _blendshapeCoefficients; - } else { - blendshapes.resize(_blendshapeCoefficients.size()); - for (int i = 0; i < _blendshapeCoefficients.size(); i++) { - blendshapes[i] = glm::mix(0.0f, _blendshapeCoefficients[i], fadeCoefficient); - } - return blendshapes; - } -} - -float FaceTracker::getFadeCoefficient() const { - return _fadeCoefficient; -} - -const glm::vec3 FaceTracker::getHeadTranslation() const { - return glm::mix(glm::vec3(0.0f), _headTranslation, getFadeCoefficient()); -} - -const glm::quat FaceTracker::getHeadRotation() const { - return safeMix(glm::quat(), _headRotation, getFadeCoefficient()); -} - -void FaceTracker::update(float deltaTime) { - // Based on exponential distributions: http://en.wikipedia.org/wiki/Exponential_distribution - static const float EPSILON = 0.02f; // MUST BE < 1.0f - static const float INVERSE_AT_EPSILON = -std::log(EPSILON); // So that f(1.0f) = EPSILON ~ 0.0f - static const float RELAXATION_TIME = 0.8f; // sec - - if (isTracking()) { - if (_relaxationStatus == 1.0f) { - _fadeCoefficient = 1.0f; - return; - } - _relaxationStatus = glm::clamp(_relaxationStatus + deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = 1.0f - std::exp(-_relaxationStatus * INVERSE_AT_EPSILON); - } else { - if (_relaxationStatus == 0.0f) { - _fadeCoefficient = 0.0f; - return; - } - _relaxationStatus = glm::clamp(_relaxationStatus - deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = std::exp(-(1.0f - _relaxationStatus) * INVERSE_AT_EPSILON); - } -} - -void FaceTracker::reset() { - if (isActive() && !_isCalculatingFPS) { - QTimer::singleShot(FPS_TIMER_DELAY, this, SLOT(startFPSTimer())); - _isCalculatingFPS = true; - } -} - -void FaceTracker::startFPSTimer() { - _frameCount = 0; - QTimer::singleShot(FPS_TIMER_DURATION, this, SLOT(finishFPSTimer())); -} - -void FaceTracker::countFrame() { - if (_isCalculatingFPS) { - _frameCount++; - } -} - -void FaceTracker::finishFPSTimer() { - qCDebug(trackers) << "Face tracker FPS =" << (float)_frameCount / ((float)FPS_TIMER_DURATION / 1000.0f); - _isCalculatingFPS = false; -} - -void FaceTracker::toggleMute() { - _isMuted = !_isMuted; - emit muteToggled(); -} - -void FaceTracker::setEyeDeflection(float eyeDeflection) { - _eyeDeflection.set(eyeDeflection); -} - -void FaceTracker::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen, float mouth2, float mouth3, float mouth4, QVector& coefficients) { - const int MMMM_BLENDSHAPE = 34; - const int FUNNEL_BLENDSHAPE = 40; - const int SMILE_LEFT_BLENDSHAPE = 28; - const int SMILE_RIGHT_BLENDSHAPE = 29; - const int MAX_FAKE_BLENDSHAPE = 40; // Largest modified blendshape from above and below - - coefficients.resize(std::max((int)coefficients.size(), MAX_FAKE_BLENDSHAPE + 1)); - qFill(coefficients.begin(), coefficients.end(), 0.0f); - coefficients[_leftBlinkIndex] = leftBlink; - coefficients[_rightBlinkIndex] = rightBlink; - coefficients[_browUpCenterIndex] = browUp; - coefficients[_browUpLeftIndex] = browUp; - coefficients[_browUpRightIndex] = browUp; - coefficients[_jawOpenIndex] = jawOpen; - coefficients[SMILE_LEFT_BLENDSHAPE] = coefficients[SMILE_RIGHT_BLENDSHAPE] = mouth4; - coefficients[MMMM_BLENDSHAPE] = mouth2; - coefficients[FUNNEL_BLENDSHAPE] = mouth3; -} - diff --git a/libraries/trackers/src/trackers/FaceTracker.h b/libraries/trackers/src/trackers/FaceTracker.h deleted file mode 100644 index 47fbf72616..0000000000 --- a/libraries/trackers/src/trackers/FaceTracker.h +++ /dev/null @@ -1,131 +0,0 @@ -// -// Created by Andrzej Kapolka on 4/9/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_FaceTracker_h -#define hifi_FaceTracker_h - -#include -#include - -#include -#include - -#include - -/// Base class for face trackers (DDE, BinaryVR). - -class FaceTracker : public QObject { - Q_OBJECT - -public: - virtual bool isActive() const { return false; } - virtual bool isTracking() const { return false; } - - virtual void init(); - virtual void update(float deltaTime); - virtual void reset(); - - float getFadeCoefficient() const; - - const glm::vec3 getHeadTranslation() const; - const glm::quat getHeadRotation() const; - - float getEstimatedEyePitch() const { return _estimatedEyePitch; } - float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } - - int getNumBlendshapes() const { return _blendshapeCoefficients.size(); } - bool isValidBlendshapeIndex(int index) const { return index >= 0 && index < getNumBlendshapes(); } - const QVector& getBlendshapeCoefficients() const; - float getBlendshapeCoefficient(int index) const; - - static bool isMuted() { return _isMuted; } - static void setIsMuted(bool isMuted) { _isMuted = isMuted; } - - static float getEyeDeflection() { return _eyeDeflection.get(); } - static void setEyeDeflection(float eyeDeflection); - - static void updateFakeCoefficients(float leftBlink, - float rightBlink, - float browUp, - float jawOpen, - float mouth2, - float mouth3, - float mouth4, - QVector& coefficients); - -signals: - - /**jsdoc - * @function FaceTracker.muteToggled - * @returns {Signal} - */ - void muteToggled(); - -public slots: - - // No JSDoc here because it's overridden in DdeFaceTracker. - virtual void setEnabled(bool enabled) = 0; - - /**jsdoc - * @function FaceTracker.toggleMute - */ - void toggleMute(); - - /**jsdoc - * @function FaceTracker.getMuted - * @returns {boolean} - */ - bool getMuted() { return _isMuted; } - -protected: - virtual ~FaceTracker() {}; - - bool _isInitialized = false; - static bool _isMuted; - - glm::vec3 _headTranslation = glm::vec3(0.0f); - glm::quat _headRotation = glm::quat(); - float _estimatedEyePitch = 0.0f; - float _estimatedEyeYaw = 0.0f; - QVector _blendshapeCoefficients; - - float _relaxationStatus = 0.0f; // Between 0.0f and 1.0f - float _fadeCoefficient = 0.0f; // Between 0.0f and 1.0f - - void countFrame(); - -private slots: - void startFPSTimer(); - void finishFPSTimer(); - -private: - bool _isCalculatingFPS = false; - int _frameCount = 0; - - // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes - static const int _leftBlinkIndex = 0; - static const int _rightBlinkIndex = 1; - static const int _leftEyeOpenIndex = 8; - static const int _rightEyeOpenIndex = 9; - - // Brows - static const int _browDownLeftIndex = 14; - static const int _browDownRightIndex = 15; - static const int _browUpCenterIndex = 16; - static const int _browUpLeftIndex = 17; - static const int _browUpRightIndex = 18; - - static const int _mouthSmileLeftIndex = 28; - static const int _mouthSmileRightIndex = 29; - - static const int _jawOpenIndex = 21; - - static Setting::Handle _eyeDeflection; -}; - -#endif // hifi_FaceTracker_h diff --git a/libraries/trackers/src/trackers/Logging.cpp b/libraries/trackers/src/trackers/Logging.cpp deleted file mode 100644 index a4dcf1b711..0000000000 --- a/libraries/trackers/src/trackers/Logging.cpp +++ /dev/null @@ -1,11 +0,0 @@ -// -// Created by Bradley Austin Davis on 2017/04/25 -// Copyright 2013-2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "Logging.h" - -Q_LOGGING_CATEGORY(trackers, "hifi.trackers") diff --git a/libraries/trackers/src/trackers/Logging.h b/libraries/trackers/src/trackers/Logging.h deleted file mode 100644 index 554429b61d..0000000000 --- a/libraries/trackers/src/trackers/Logging.h +++ /dev/null @@ -1,16 +0,0 @@ -// -// Created by Bradley Austin Davis on 2017/04/25 -// Copyright 2013-2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_TrackersLogging_h -#define hifi_TrackersLogging_h - -#include - -Q_DECLARE_LOGGING_CATEGORY(trackers) - -#endif // hifi_TrackersLogging_h diff --git a/tools/jsdoc/plugins/hifi.js b/tools/jsdoc/plugins/hifi.js index 07549530ce..8b18d02458 100644 --- a/tools/jsdoc/plugins/hifi.js +++ b/tools/jsdoc/plugins/hifi.js @@ -29,7 +29,6 @@ exports.handlers = { '../../interface/src/audio', '../../interface/src/avatar', '../../interface/src/commerce', - '../../interface/src/devices', '../../interface/src/java', '../../interface/src/networking', '../../interface/src/raypick', @@ -64,7 +63,6 @@ exports.handlers = { '../../libraries/shared/src', '../../libraries/shared/src/shared', '../../libraries/task/src/task', - '../../libraries/trackers/src/trackers', '../../libraries/ui/src', '../../libraries/ui/src/ui', '../../plugins/oculus/src', From 6efd74a339189433ea086807dfe05f41e3997a61 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 11 Oct 2019 09:37:01 -0700 Subject: [PATCH 03/19] WIP adding blendshapes to input system --- interface/resources/controllers/standard.json | 1 + .../controllers/standard_nomovement.json | 1 + interface/resources/controllers/vive.json | 2 + interface/src/avatar/MyHead.cpp | 10 ++- .../src/avatars-renderer/Head.cpp | 18 ++--- .../controllers/src/controllers/Actions.cpp | 68 ++++++++++++++++- .../controllers/src/controllers/Actions.h | 68 ++++++++++++++++- .../src/controllers/StandardController.cpp | 68 ++++++++++++++++- .../src/controllers/StandardControls.h | 3 + libraries/shared/src/BlendshapeConstants.cpp | 51 ++++++------- libraries/shared/src/BlendshapeConstants.h | 74 +++++++++++++++---- 11 files changed, 300 insertions(+), 64 deletions(-) diff --git a/interface/resources/controllers/standard.json b/interface/resources/controllers/standard.json index 195f909942..7f24924f7b 100644 --- a/interface/resources/controllers/standard.json +++ b/interface/resources/controllers/standard.json @@ -166,6 +166,7 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, + // AJT: blendshapes { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, diff --git a/interface/resources/controllers/standard_nomovement.json b/interface/resources/controllers/standard_nomovement.json index 602d3bb798..67eddfad31 100644 --- a/interface/resources/controllers/standard_nomovement.json +++ b/interface/resources/controllers/standard_nomovement.json @@ -61,6 +61,7 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, + // AJT: blendshapes { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, diff --git a/interface/resources/controllers/vive.json b/interface/resources/controllers/vive.json index b6fae1dd79..045f72739c 100644 --- a/interface/resources/controllers/vive.json +++ b/interface/resources/controllers/vive.json @@ -98,6 +98,8 @@ { "from": "Vive.Head", "to" : "Standard.Head" }, { "from": "Vive.LeftEye", "to" : "Standard.LeftEye" }, { "from": "Vive.RightEye", "to" : "Standard.RightEye" }, + + // AJT: blendshapes (only keep blink) { "from": "Vive.LeftEyeBlink", "to" : "Standard.LeftEyeBlink" }, { "from": "Vive.RightEyeBlink", "to" : "Standard.RightEyeBlink" }, diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 1c6e4690dd..73259fe073 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -58,6 +58,8 @@ void MyHead::simulate(float deltaTime) { // } // } + // AJT: blendshapes + auto userInputMapper = DependencyManager::get(); bool eyeLidsTracked = userInputMapper->getActionStateValid(controller::Action::LEFT_EYE_BLINK) && @@ -69,13 +71,13 @@ void MyHead::simulate(float deltaTime) { float leftEyeBlink = userInputMapper->getActionState(controller::Action::LEFT_EYE_BLINK); float rightEyeBlink = userInputMapper->getActionState(controller::Action::RIGHT_EYE_BLINK); _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[EYE_BLINK_INDICES[0]] = leftEyeBlink; - _blendshapeCoefficients[EYE_BLINK_INDICES[1]] = rightEyeBlink; + _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = leftEyeBlink; + _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = rightEyeBlink; } else { const float FULLY_OPEN = 0.0f; _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[EYE_BLINK_INDICES[0]] = FULLY_OPEN; - _blendshapeCoefficients[EYE_BLINK_INDICES[1]] = FULLY_OPEN; + _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = FULLY_OPEN; + _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = FULLY_OPEN; } } Parent::simulate(deltaTime); diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index ea6cdd7087..c7f807509c 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -262,26 +262,26 @@ void Head::applyEyelidOffset(glm::quat headOrientation) { float blinkUpCoefficient = -eyelidOffset; float blinkDownCoefficient = BLINK_DOWN_MULTIPLIER * eyelidOffset; - + float openUpCoefficient = eyelidOffset; float openDownCoefficient = OPEN_DOWN_MULTIPLIER * eyelidOffset; - + float browsUpCoefficient = BROW_UP_MULTIPLIER * eyelidOffset; float browsDownCoefficient = 0.0f; bool isLookingUp = (eyePitch > 0); - + if (isLookingUp) { for (int i = 0; i < 2; i++) { - _transientBlendshapeCoefficients[EYE_BLINK_INDICES[i]] = blinkUpCoefficient; - _transientBlendshapeCoefficients[EYE_OPEN_INDICES[i]] = openUpCoefficient; - _transientBlendshapeCoefficients[BROWS_U_INDICES[i]] = browsUpCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::EyeBlink_L + i] = blinkUpCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::EyeOpen_L + i] = openUpCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::BrowsU_L + i] = browsUpCoefficient; } } else { for (int i = 0; i < 2; i++) { - _transientBlendshapeCoefficients[EYE_BLINK_INDICES[i]] = blinkDownCoefficient; - _transientBlendshapeCoefficients[EYE_OPEN_INDICES[i]] = openDownCoefficient; - _transientBlendshapeCoefficients[BROWS_U_INDICES[i]] = browsDownCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::EyeBlink_L + i] = blinkDownCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::EyeOpen_L + i] = openDownCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::BrowsU_L + i] = browsDownCoefficient; } } } diff --git a/libraries/controllers/src/controllers/Actions.cpp b/libraries/controllers/src/controllers/Actions.cpp index b6b96216b5..198c342b1d 100644 --- a/libraries/controllers/src/controllers/Actions.cpp +++ b/libraries/controllers/src/controllers/Actions.cpp @@ -349,8 +349,72 @@ namespace controller { makePosePair(Action::HEAD, "Head"), makePosePair(Action::LEFT_EYE, "LeftEye"), makePosePair(Action::RIGHT_EYE, "RightEye"), - makeAxisPair(Action::LEFT_EYE_BLINK, "LeftEyeBlink"), - makeAxisPair(Action::RIGHT_EYE_BLINK, "RightEyeBlink"), + + // AJT: blendshapes + makeAxisPair(Action::EYEBLINK_L, "EyeBlink_L"), + makeAxisPair(Action::EYEBLINK_R, "EyeBlink_R"), + makeAxisPair(Action::EYESQUINT_L, "EyeSquint_L"), + makeAxisPair(Action::EYESQUINT_R, "EyeSquint_R"), + makeAxisPair(Action::EYEDOWN_L, "EyeDown_L"), + makeAxisPair(Action::EYEDOWN_R, "EyeDown_R"), + makeAxisPair(Action::EYEIN_L, "EyeIn_L"), + makeAxisPair(Action::EYEIN_R, "EyeIn_R"), + makeAxisPair(Action::EYEOPEN_L, "EyeOpen_L"), + makeAxisPair(Action::EYEOPEN_R, "EyeOpen_R"), + makeAxisPair(Action::EYEOUT_L, "EyeOut_L"), + makeAxisPair(Action::EYEOUT_R, "EyeOut_R"), + makeAxisPair(Action::EYEUP_L, "EyeUp_L"), + makeAxisPair(Action::EYEUP_R, "EyeUp_R"), + makeAxisPair(Action::BROWSD_L, "BrowsD_L"), + makeAxisPair(Action::BROWSD_R, "BrowsD_R"), + makeAxisPair(Action::BROWSU_C, "BrowsU_C"), + makeAxisPair(Action::BROWSU_L, "BrowsU_L"), + makeAxisPair(Action::BROWSU_R, "BrowsU_R"), + makeAxisPair(Action::JAWFWD, "JawFwd"), + makeAxisPair(Action::JAWLEFT, "JawLeft"), + makeAxisPair(Action::JAWOPEN, "JawOpen"), + makeAxisPair(Action::JAWRIGHT, "JawRight"), + makeAxisPair(Action::MOUTHLEFT, "MouthLeft"), + makeAxisPair(Action::MOUTHRIGHT, "MouthRight"), + makeAxisPair(Action::MOUTHFROWN_L, "MouthFrown_L"), + makeAxisPair(Action::MOUTHFROWN_R, "MouthFrown_R"), + makeAxisPair(Action::MOUTHSMILE_L, "MouthSmile_L"), + makeAxisPair(Action::MOUTHSMILE_R, "MouthSmile_R"), + makeAxisPair(Action::MOUTHDIMPLE_L, "MouthDimple_L"), + makeAxisPair(Action::MOUTHDIMPLE_R, "MouthDimple_R"), + makeAxisPair(Action::LIPSSTRETCH_L, "LipsStretch_L"), + makeAxisPair(Action::LIPSSTRETCH_R, "LipsStretch_R"), + makeAxisPair(Action::LIPSUPPERCLOSE, "LipsUpperClose"), + makeAxisPair(Action::LIPSLOWERCLOSE, "LipsLowerClose"), + makeAxisPair(Action::LIPSUPPEROPEN, "LipsUpperOpen"), + makeAxisPair(Action::LIPSLOWEROPEN, "LipsLowerOpen"), + makeAxisPair(Action::LIPSFUNNEL, "LipsFunnel"), + makeAxisPair(Action::LIPSPUCKER, "LipsPucker"), + makeAxisPair(Action::PUFF, "Puff"), + makeAxisPair(Action::CHEEKSQUINT_L, "CheekSquint_L"), + makeAxisPair(Action::CHEEKSQUINT_R, "CheekSquint_R"), + makeAxisPair(Action::LIPSTOGETHER, "LipsTogether"), + makeAxisPair(Action::MOUTHUPPERUP_L, "MouthUpperUp_L"), + makeAxisPair(Action::MOUTHUPPERUP_R, "MouthUpperUp_R"), + makeAxisPair(Action::MOUTHLOWERDOWN_L, "MouthLowerDown_L"), + makeAxisPair(Action::MOUTHLOWERDOWN_R, "MouthLowerDown_R"), + makeAxisPair(Action::MOUTHPRESS_L, "MouthPress_L"), + makeAxisPair(Action::MOUTHPRESS_R, "MouthPress_R"), + makeAxisPair(Action::MOUTHSHRUGLOWER, "MouthShrugLower"), + makeAxisPair(Action::MOUTHSHRUGUPPER, "MouthShrugUpper"), + makeAxisPair(Action::NOSESNEER_L, "NoseSneer_L"), + makeAxisPair(Action::NOSESNEER_R, "NoseSneer_R"), + makeAxisPair(Action::TONGUEOUT, "TongueOut"), + makeAxisPair(Action::USERBLENDSHAPE0, "UserBlendshape0"), + makeAxisPair(Action::USERBLENDSHAPE1, "UserBlendshape1"), + makeAxisPair(Action::USERBLENDSHAPE2, "UserBlendshape2"), + makeAxisPair(Action::USERBLENDSHAPE3, "UserBlendshape3"), + makeAxisPair(Action::USERBLENDSHAPE4, "UserBlendshape4"), + makeAxisPair(Action::USERBLENDSHAPE5, "UserBlendshape5"), + makeAxisPair(Action::USERBLENDSHAPE6, "UserBlendshape6"), + makeAxisPair(Action::USERBLENDSHAPE7, "UserBlendshape7"), + makeAxisPair(Action::USERBLENDSHAPE8, "UserBlendshape8"), + makeAxisPair(Action::USERBLENDSHAPE9, "UserBlendshape9"), makePosePair(Action::LEFT_HAND_THUMB1, "LeftHandThumb1"), makePosePair(Action::LEFT_HAND_THUMB2, "LeftHandThumb2"), diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index f91d9f2522..e1ad17aafa 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -183,8 +183,72 @@ enum class Action { LEFT_EYE, RIGHT_EYE, - LEFT_EYE_BLINK, - RIGHT_EYE_BLINK, + + // AJT: blendshapes + EyeBlink_L, + EyeBlink_R, + EyeSquint_L, + EyeSquint_R, + EyeDown_L, + EyeDown_R, + EyeIn_L, + EyeIn_R, + EyeOpen_L, + EyeOpen_R, + EyeOut_L, + EyeOut_R, + EyeUp_L, + EyeUp_R, + BrowsD_L, + BrowsD_R, + BrowsU_C, + BrowsU_L, + BrowsU_R, + JawFwd, + JawLeft, + JawOpen, + JawRight, + MouthLeft, + MouthRight, + MouthFrown_L, + MouthFrown_R, + MouthSmile_L, + MouthSmile_R, + MouthDimple_L, + MouthDimple_R, + LipsStretch_L, + LipsStretch_R, + LipsUpperClose, + LipsLowerClose, + LipsUpperOpen, + LipsLowerOpen, + LipsFunnel, + LipsPucker, + Puff, + CheekSquint_L, + CheekSquint_R, + LipsTogether, + MouthUpperUp_L, + MouthUpperUp_R, + MouthLowerDown_L, + MouthLowerDown_R, + MouthPress_L, + MouthPress_R, + MouthShrugLower, + MouthShrugUpper, + NoseSneer_L, + NoseSneer_R, + TongueOut, + UserBlendshape0, + UserBlendshape1, + UserBlendshape2, + UserBlendshape3, + UserBlendshape4, + UserBlendshape5, + UserBlendshape6, + UserBlendshape7, + UserBlendshape8, + UserBlendshape9, NUM_ACTIONS }; diff --git a/libraries/controllers/src/controllers/StandardController.cpp b/libraries/controllers/src/controllers/StandardController.cpp index ae592485dc..14697b26d2 100644 --- a/libraries/controllers/src/controllers/StandardController.cpp +++ b/libraries/controllers/src/controllers/StandardController.cpp @@ -355,8 +355,72 @@ Input::NamedVector StandardController::getAvailableInputs() const { makePair(HEAD, "Head"), makePair(LEFT_EYE, "LeftEye"), makePair(RIGHT_EYE, "RightEye"), - makePair(LEFT_EYE_BLINK, "LeftEyeBlink"), - makePair(RIGHT_EYE_BLINK, "RightEyeBlink"), + + // AJT: blendshapes + makePair(EYEBLINK_L, "EyeBlink_L"), + makePair(EYEBLINK_R, "EyeBlink_R"), + makePair(EYESQUINT_L, "EyeSquint_L"), + makePair(EYESQUINT_R, "EyeSquint_R"), + makePair(EYEDOWN_L, "EyeDown_L"), + makePair(EYEDOWN_R, "EyeDown_R"), + makePair(EYEIN_L, "EyeIn_L"), + makePair(EYEIN_R, "EyeIn_R"), + makePair(EYEOPEN_L, "EyeOpen_L"), + makePair(EYEOPEN_R, "EyeOpen_R"), + makePair(EYEOUT_L, "EyeOut_L"), + makePair(EYEOUT_R, "EyeOut_R"), + makePair(EYEUP_L, "EyeUp_L"), + makePair(EYEUP_R, "EyeUp_R"), + makePair(BROWSD_L, "BrowsD_L"), + makePair(BROWSD_R, "BrowsD_R"), + makePair(BROWSU_C, "BrowsU_C"), + makePair(BROWSU_L, "BrowsU_L"), + makePair(BROWSU_R, "BrowsU_R"), + makePair(JAWFWD, "JawFwd"), + makePair(JAWLEFT, "JawLeft"), + makePair(JAWOPEN, "JawOpen"), + makePair(JAWRIGHT, "JawRight"), + makePair(MOUTHLEFT, "MouthLeft"), + makePair(MOUTHRIGHT, "MouthRight"), + makePair(MOUTHFROWN_L, "MouthFrown_L"), + makePair(MOUTHFROWN_R, "MouthFrown_R"), + makePair(MOUTHSMILE_L, "MouthSmile_L"), + makePair(MOUTHSMILE_R, "MouthSmile_R"), + makePair(MOUTHDIMPLE_L, "MouthDimple_L"), + makePair(MOUTHDIMPLE_R, "MouthDimple_R"), + makePair(LIPSSTRETCH_L, "LipsStretch_L"), + makePair(LIPSSTRETCH_R, "LipsStretch_R"), + makePair(LIPSUPPERCLOSE, "LipsUpperClose"), + makePair(LIPSLOWERCLOSE, "LipsLowerClose"), + makePair(LIPSUPPEROPEN, "LipsUpperOpen"), + makePair(LIPSLOWEROPEN, "LipsLowerOpen"), + makePair(LIPSFUNNEL, "LipsFunnel"), + makePair(LIPSPUCKER, "LipsPucker"), + makePair(PUFF, "Puff"), + makePair(CHEEKSQUINT_L, "CheekSquint_L"), + makePair(CHEEKSQUINT_R, "CheekSquint_R"), + makePair(LIPSTOGETHER, "LipsTogether"), + makePair(MOUTHUPPERUP_L, "MouthUpperUp_L"), + makePair(MOUTHUPPERUP_R, "MouthUpperUp_R"), + makePair(MOUTHLOWERDOWN_L, "MouthLowerDown_L"), + makePair(MOUTHLOWERDOWN_R, "MouthLowerDown_R"), + makePair(MOUTHPRESS_L, "MouthPress_L"), + makePair(MOUTHPRESS_R, "MouthPress_R"), + makePair(MOUTHSHRUGLOWER, "MouthShrugLower"), + makePair(MOUTHSHRUGUPPER, "MouthShrugUpper"), + makePair(NOSESNEER_L, "NoseSneer_L"), + makePair(NOSESNEER_R, "NoseSneer_R"), + makePair(TONGUEOUT, "TongueOut"), + makePair(USERBLENDSHAPE0, "UserBlendshape0"), + makePair(USERBLENDSHAPE1, "UserBlendshape1"), + makePair(USERBLENDSHAPE2, "UserBlendshape2"), + makePair(USERBLENDSHAPE3, "UserBlendshape3"), + makePair(USERBLENDSHAPE4, "UserBlendshape4"), + makePair(USERBLENDSHAPE5, "UserBlendshape5"), + makePair(USERBLENDSHAPE6, "UserBlendshape6"), + makePair(USERBLENDSHAPE7, "UserBlendshape7"), + makePair(USERBLENDSHAPE8, "UserBlendshape8"), + makePair(USERBLENDSHAPE9, "UserBlendshape9"), // Aliases, PlayStation style names makePair(LB, "L1"), diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index 99d9246264..2af822ffc2 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -90,8 +90,11 @@ namespace controller { // Grips LEFT_GRIP, RIGHT_GRIP, + + // AJT: blendshapes LEFT_EYE_BLINK, RIGHT_EYE_BLINK, + NUM_STANDARD_AXES, LZ = LT, RZ = RT diff --git a/libraries/shared/src/BlendshapeConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp index 528b941b73..7564c31944 100644 --- a/libraries/shared/src/BlendshapeConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -34,7 +34,6 @@ const char* FACESHIFT_BLENDSHAPES[] = { "JawFwd", "JawLeft", "JawOpen", - "JawChew", "JawRight", "MouthLeft", "MouthRight", @@ -48,40 +47,34 @@ const char* FACESHIFT_BLENDSHAPES[] = { "LipsStretch_R", "LipsUpperClose", "LipsLowerClose", - "LipsUpperUp", - "LipsLowerDown", "LipsUpperOpen", "LipsLowerOpen", "LipsFunnel", "LipsPucker", - "ChinLowerRaise", - "ChinUpperRaise", - "Sneer", "Puff", "CheekSquint_L", "CheekSquint_R", + "LipsTogether", + "MouthUpperUp_L", + "MouthUpperUp_R", + "MouthLowerDown_L", + "MouthLowerDown_R", + "MouthPress_L", + "MouthPress_R", + "MouthShrugLower", + "MouthShrugUpper", + "NoseSneer_L", + "NoseSneer_R", + "TongueOut", + "UserBlendshape0", + "UserBlendshape1", + "UserBlendshape2", + "UserBlendshape3", + "UserBlendshape4", + "UserBlendshape5", + "UserBlendshape6", + "UserBlendshape7", + "UserBlendshape8", + "UserBlendshape9", "" }; - -// new in ARKit -// LipsTogether -// MouthPressLeft -// MouthPressRight -// MouthShrugLower -// MouthShrugUpper -// TongueOut - -const int EYE_BLINK_L_INDEX = 0; -const int EYE_BLINK_R_INDEX = 1; -const int EYE_SQUINT_L_INDEX = 2; -const int EYE_SQUINT_R_INDEX = 3; -const int EYE_OPEN_L_INDEX = 8; -const int EYE_OPEN_R_INDEX = 9; -const int BROWS_U_L_INDEX = 17; -const int BROWS_U_R_INDEX = 18; - - -const int EYE_BLINK_INDICES[] = { EYE_BLINK_L_INDEX, EYE_BLINK_R_INDEX }; -const int EYE_SQUINT_INDICES[] = { EYE_SQUINT_L_INDEX, EYE_SQUINT_R_INDEX }; -const int EYE_OPEN_INDICES[] = { EYE_OPEN_L_INDEX, EYE_OPEN_R_INDEX }; -const int BROWS_U_INDICES[] = { BROWS_U_L_INDEX, BROWS_U_R_INDEX }; diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 0fca03e34b..97b459fa0e 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -15,12 +15,6 @@ /// The names of the blendshapes expected by Faceshift, terminated with an empty string. extern const char* FACESHIFT_BLENDSHAPES[]; -// Eyes and Brows indices -extern const int EYE_BLINK_INDICES[]; -extern const int EYE_OPEN_INDICES[]; -extern const int BROWS_U_INDICES[]; -extern const int EYE_SQUINT_INDICES[]; - enum class Blendshapes : int { EyeBlink_L = 0, EyeBlink_R, @@ -32,7 +26,7 @@ enum class Blendshapes : int { EyeIn_R, EyeOpen_L, EyeOpen_R, - EyeOut_L, // 10 + EyeOut_L, EyeOut_R, EyeUp_L, EyeUp_R, @@ -42,9 +36,8 @@ enum class Blendshapes : int { BrowsU_L, BrowsU_R, JawFwd, - JawLeft, // 20 + JawLeft, JawOpen, - JawChew, // legacy not in ARKit JawRight, MouthLeft, MouthRight, @@ -52,25 +45,74 @@ enum class Blendshapes : int { MouthFrown_R, MouthSmile_L, MouthSmile_R, - MouthDimple_L, // 30 + MouthDimple_L, MouthDimple_R, LipsStretch_L, LipsStretch_R, LipsUpperClose, LipsLowerClose, - LipsUpperUp, // legacy, split in ARKit - LipsLowerDown, // legacy, split in ARKit LipsUpperOpen, LipsLowerOpen, - LipsFunnel, // 40 + LipsFunnel, LipsPucker, - ChinLowerRaise, - ChinUpperRaise, - Sneer, // legacy, split in ARKit Puff, CheekSquint_L, CheekSquint_R, + LipsTogether, + MouthUpperUp_L, + MouthUpperUp_R, + MouthLowerDown_L, + MouthLowerDown_R, + MouthPress_L, + MouthPress_R, + MouthShrugLower, + MouthShrugUpper, + NoseSneer_L, + NoseSneer_R, + TongueOut, + UserBlendshape0, + UserBlendshape1, + UserBlendshape2, + UserBlendshape3, + UserBlendshape4, + UserBlendshape5, + UserBlendshape6, + UserBlendshape7, + UserBlendshape8, + UserBlendshape9, BlendshapeCount }; +enum class LegacyBlendshpaes : int { + JawChew, // not in ARKit + LipsUpperUp, // split in ARKit + LipsLowerDown, // split in ARKit + ChinLowerRaise, // not in ARKit + ChinUpperRaise, // not in ARKit + Sneer, // split in ARKit + LegacyBlendshapeCount +} + +// NEW in ARKit +// * LipsTogether +// * MouthUpperUp_L +// * MouthUpperUp_R +// * MouthLowerDown_L +// * MouthLowerDown_R +// * MouthPress_L +// * MouthPress_R +// * MouthShrugLower +// * MouthShrugUpper +// * NoseSneer_L +// * NoseSneer_R +// * TongueOut + +// Legacy shapes +// * JawChew (not in ARKit) +// * MouthUpperUp (split in ARKit) +// * MouthLowerDown (split in ARKit) +// * Sneer (split in ARKit) +// * ChinLowerRaise (not in ARKit) +// * ChinUpperRase (not in ARKit) + #endif // hifi_BlendshapeConstants_h From 389f5a1d335ad4c88f358fb8ff7356a74741cac6 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 15 Oct 2019 11:37:10 -0700 Subject: [PATCH 04/19] ARKit Blendshape hookup work --- interface/resources/controllers/standard.json | 68 ++++++++- .../controllers/standard_nomovement.json | 67 ++++++++- interface/resources/controllers/vive.json | 5 +- interface/src/avatar/MyAvatar.cpp | 12 -- interface/src/avatar/MyHead.cpp | 142 ++++++++++++++---- interface/src/avatar/MySkeletonModel.cpp | 5 + .../src/avatars-renderer/Head.cpp | 12 +- .../src/avatars-renderer/SkeletonModel.cpp | 4 + libraries/avatars/src/AvatarData.cpp | 79 ++++++---- libraries/avatars/src/AvatarData.h | 11 +- libraries/avatars/src/HeadData.cpp | 60 ++++---- libraries/avatars/src/HeadData.h | 43 ++++-- .../controllers/src/controllers/Actions.h | 128 ++++++++-------- .../src/controllers/StandardControls.h | 66 +++++++- .../networking/src/udt/PacketHeaders.cpp | 4 +- libraries/networking/src/udt/PacketHeaders.h | 3 +- libraries/shared/src/BlendshapeConstants.h | 2 +- 17 files changed, 507 insertions(+), 204 deletions(-) diff --git a/interface/resources/controllers/standard.json b/interface/resources/controllers/standard.json index 7f24924f7b..8c84039b86 100644 --- a/interface/resources/controllers/standard.json +++ b/interface/resources/controllers/standard.json @@ -166,10 +166,70 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, - // AJT: blendshapes - { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, - { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, - + { "from": "Standard.EyeBlink_L", "to": "Actions.EyeBlink_L" }, + { "from": "Standard.EyeBlink_R", "to": "Actions.EyeBlink_R" }, + { "from": "Standard.EyeSquint_L", "to": "Actions.EyeSquint_L" }, + { "from": "Standard.EyeSquint_R", "to": "Actions.EyeSquint_R" }, + { "from": "Standard.EyeDown_L", "to": "Actions.EyeDown_L" }, + { "from": "Standard.EyeDown_R", "to": "Actions.EyeDown_R" }, + { "from": "Standard.EyeIn_L", "to": "Actions.EyeIn_L" }, + { "from": "Standard.EyeIn_R", "to": "Actions.EyeIn_R" }, + { "from": "Standard.EyeOpen_L", "to": "Actions.EyeOpen_L" }, + { "from": "Standard.EyeOpen_R", "to": "Actions.EyeOpen_R" }, + { "from": "Standard.EyeOut_L", "to": "Actions.EyeOut_L" }, + { "from": "Standard.EyeOut_R", "to": "Actions.EyeOut_R" }, + { "from": "Standard.EyeUp_L", "to": "Actions.EyeUp_L" }, + { "from": "Standard.EyeUp_R", "to": "Actions.EyeUp_R" }, + { "from": "Standard.BrowsD_L", "to": "Actions.BrowsD_L" }, + { "from": "Standard.BrowsD_R", "to": "Actions.BrowsD_R" }, + { "from": "Standard.BrowsU_C", "to": "Actions.BrowsU_C" }, + { "from": "Standard.BrowsU_L", "to": "Actions.BrowsU_L" }, + { "from": "Standard.BrowsU_R", "to": "Actions.BrowsU_R" }, + { "from": "Standard.JawFwd", "to": "Actions.JawFwd" }, + { "from": "Standard.JawLeft", "to": "Actions.JawLeft" }, + { "from": "Standard.JawOpen", "to": "Actions.JawOpen" }, + { "from": "Standard.JawRight", "to": "Actions.JawRight" }, + { "from": "Standard.MouthLeft", "to": "Actions.MouthLeft" }, + { "from": "Standard.MouthRight", "to": "Actions.MouthRight" }, + { "from": "Standard.MouthFrown_L", "to": "Actions.MouthFrown_L" }, + { "from": "Standard.MouthFrown_R", "to": "Actions.MouthFrown_R" }, + { "from": "Standard.MouthSmile_L", "to": "Actions.MouthSmile_L" }, + { "from": "Standard.MouthSmile_R", "to": "Actions.MouthSmile_R" }, + { "from": "Standard.MouthDimple_L", "to": "Actions.MouthDimple_L" }, + { "from": "Standard.MouthDimple_R", "to": "Actions.MouthDimple_R" }, + { "from": "Standard.LipsStretch_L", "to": "Actions.LipsStretch_L" }, + { "from": "Standard.LipsStretch_R", "to": "Actions.LipsStretch_R" }, + { "from": "Standard.LipsUpperClose", "to": "Actions.LipsUpperClose" }, + { "from": "Standard.LipsLowerClose", "to": "Actions.LipsLowerClose" }, + { "from": "Standard.LipsUpperOpen", "to": "Actions.LipsUpperOpen" }, + { "from": "Standard.LipsLowerOpen", "to": "Actions.LipsLowerOpen" }, + { "from": "Standard.LipsFunnel", "to": "Actions.LipsFunnel" }, + { "from": "Standard.LipsPucker", "to": "Actions.LipsPucker" }, + { "from": "Standard.Puff", "to": "Actions.Puff" }, + { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, + { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, + { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, + { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, + { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, + { "from": "Standard.MouthLowerDown_R", "to": "Actions.MouthLowerDown_R" }, + { "from": "Standard.MouthPress_L", "to": "Actions.MouthPress_L" }, + { "from": "Standard.MouthPress_R", "to": "Actions.MouthPress_R" }, + { "from": "Standard.MouthShrugLower", "to": "Actions.MouthShrugLower" }, + { "from": "Standard.MouthShrugUpper", "to": "Actions.MouthShrugUpper" }, + { "from": "Standard.NoseSneer_L", "to": "Actions.NoseSneer_L" }, + { "from": "Standard.NoseSneer_R", "to": "Actions.NoseSneer_R" }, + { "from": "Standard.TongueOut", "to": "Actions.TongueOut" }, + { "from": "Standard.UserBlendshape0", "to": "Actions.UserBlendshape0" }, + { "from": "Standard.UserBlendshape1", "to": "Actions.UserBlendshape1" }, + { "from": "Standard.UserBlendshape2", "to": "Actions.UserBlendshape2" }, + { "from": "Standard.UserBlendshape3", "to": "Actions.UserBlendshape3" }, + { "from": "Standard.UserBlendshape4", "to": "Actions.UserBlendshape4" }, + { "from": "Standard.UserBlendshape5", "to": "Actions.UserBlendshape5" }, + { "from": "Standard.UserBlendshape6", "to": "Actions.UserBlendshape6" }, + { "from": "Standard.UserBlendshape7", "to": "Actions.UserBlendshape7" }, + { "from": "Standard.UserBlendshape8", "to": "Actions.UserBlendshape8" }, + { "from": "Standard.UserBlendshape9", "to": "Actions.UserBlendshape9" }, { "from": "Standard.TrackedObject00", "to" : "Actions.TrackedObject00" }, { "from": "Standard.TrackedObject01", "to" : "Actions.TrackedObject01" }, diff --git a/interface/resources/controllers/standard_nomovement.json b/interface/resources/controllers/standard_nomovement.json index 67eddfad31..e311a5fa62 100644 --- a/interface/resources/controllers/standard_nomovement.json +++ b/interface/resources/controllers/standard_nomovement.json @@ -61,9 +61,70 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, - // AJT: blendshapes - { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, - { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, + { "from": "Standard.EyeBlink_L", "to": "Actions.EyeBlink_L" }, + { "from": "Standard.EyeBlink_R", "to": "Actions.EyeBlink_R" }, + { "from": "Standard.EyeSquint_L", "to": "Actions.EyeSquint_L" }, + { "from": "Standard.EyeSquint_R", "to": "Actions.EyeSquint_R" }, + { "from": "Standard.EyeDown_L", "to": "Actions.EyeDown_L" }, + { "from": "Standard.EyeDown_R", "to": "Actions.EyeDown_R" }, + { "from": "Standard.EyeIn_L", "to": "Actions.EyeIn_L" }, + { "from": "Standard.EyeIn_R", "to": "Actions.EyeIn_R" }, + { "from": "Standard.EyeOpen_L", "to": "Actions.EyeOpen_L" }, + { "from": "Standard.EyeOpen_R", "to": "Actions.EyeOpen_R" }, + { "from": "Standard.EyeOut_L", "to": "Actions.EyeOut_L" }, + { "from": "Standard.EyeOut_R", "to": "Actions.EyeOut_R" }, + { "from": "Standard.EyeUp_L", "to": "Actions.EyeUp_L" }, + { "from": "Standard.EyeUp_R", "to": "Actions.EyeUp_R" }, + { "from": "Standard.BrowsD_L", "to": "Actions.BrowsD_L" }, + { "from": "Standard.BrowsD_R", "to": "Actions.BrowsD_R" }, + { "from": "Standard.BrowsU_C", "to": "Actions.BrowsU_C" }, + { "from": "Standard.BrowsU_L", "to": "Actions.BrowsU_L" }, + { "from": "Standard.BrowsU_R", "to": "Actions.BrowsU_R" }, + { "from": "Standard.JawFwd", "to": "Actions.JawFwd" }, + { "from": "Standard.JawLeft", "to": "Actions.JawLeft" }, + { "from": "Standard.JawOpen", "to": "Actions.JawOpen" }, + { "from": "Standard.JawRight", "to": "Actions.JawRight" }, + { "from": "Standard.MouthLeft", "to": "Actions.MouthLeft" }, + { "from": "Standard.MouthRight", "to": "Actions.MouthRight" }, + { "from": "Standard.MouthFrown_L", "to": "Actions.MouthFrown_L" }, + { "from": "Standard.MouthFrown_R", "to": "Actions.MouthFrown_R" }, + { "from": "Standard.MouthSmile_L", "to": "Actions.MouthSmile_L" }, + { "from": "Standard.MouthSmile_R", "to": "Actions.MouthSmile_R" }, + { "from": "Standard.MouthDimple_L", "to": "Actions.MouthDimple_L" }, + { "from": "Standard.MouthDimple_R", "to": "Actions.MouthDimple_R" }, + { "from": "Standard.LipsStretch_L", "to": "Actions.LipsStretch_L" }, + { "from": "Standard.LipsStretch_R", "to": "Actions.LipsStretch_R" }, + { "from": "Standard.LipsUpperClose", "to": "Actions.LipsUpperClose" }, + { "from": "Standard.LipsLowerClose", "to": "Actions.LipsLowerClose" }, + { "from": "Standard.LipsUpperOpen", "to": "Actions.LipsUpperOpen" }, + { "from": "Standard.LipsLowerOpen", "to": "Actions.LipsLowerOpen" }, + { "from": "Standard.LipsFunnel", "to": "Actions.LipsFunnel" }, + { "from": "Standard.LipsPucker", "to": "Actions.LipsPucker" }, + { "from": "Standard.Puff", "to": "Actions.Puff" }, + { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, + { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, + { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, + { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, + { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, + { "from": "Standard.MouthLowerDown_R", "to": "Actions.MouthLowerDown_R" }, + { "from": "Standard.MouthPress_L", "to": "Actions.MouthPress_L" }, + { "from": "Standard.MouthPress_R", "to": "Actions.MouthPress_R" }, + { "from": "Standard.MouthShrugLower", "to": "Actions.MouthShrugLower" }, + { "from": "Standard.MouthShrugUpper", "to": "Actions.MouthShrugUpper" }, + { "from": "Standard.NoseSneer_L", "to": "Actions.NoseSneer_L" }, + { "from": "Standard.NoseSneer_R", "to": "Actions.NoseSneer_R" }, + { "from": "Standard.TongueOut", "to": "Actions.TongueOut" }, + { "from": "Standard.UserBlendshape0", "to": "Actions.UserBlendshape0" }, + { "from": "Standard.UserBlendshape1", "to": "Actions.UserBlendshape1" }, + { "from": "Standard.UserBlendshape2", "to": "Actions.UserBlendshape2" }, + { "from": "Standard.UserBlendshape3", "to": "Actions.UserBlendshape3" }, + { "from": "Standard.UserBlendshape4", "to": "Actions.UserBlendshape4" }, + { "from": "Standard.UserBlendshape5", "to": "Actions.UserBlendshape5" }, + { "from": "Standard.UserBlendshape6", "to": "Actions.UserBlendshape6" }, + { "from": "Standard.UserBlendshape7", "to": "Actions.UserBlendshape7" }, + { "from": "Standard.UserBlendshape8", "to": "Actions.UserBlendshape8" }, + { "from": "Standard.UserBlendshape9", "to": "Actions.UserBlendshape9" }, { "from": "Standard.TrackedObject00", "to" : "Actions.TrackedObject00" }, { "from": "Standard.TrackedObject01", "to" : "Actions.TrackedObject01" }, diff --git a/interface/resources/controllers/vive.json b/interface/resources/controllers/vive.json index 045f72739c..4090256418 100644 --- a/interface/resources/controllers/vive.json +++ b/interface/resources/controllers/vive.json @@ -99,9 +99,8 @@ { "from": "Vive.LeftEye", "to" : "Standard.LeftEye" }, { "from": "Vive.RightEye", "to" : "Standard.RightEye" }, - // AJT: blendshapes (only keep blink) - { "from": "Vive.LeftEyeBlink", "to" : "Standard.LeftEyeBlink" }, - { "from": "Vive.RightEyeBlink", "to" : "Standard.RightEyeBlink" }, + { "from": "Vive.EyeBlink_L", "to" : "Standard.EyeBlink_L" }, + { "from": "Vive.EyeBlink_R", "to" : "Standard.EyeBlink_R" }, { "from": "Vive.LeftFoot", "to" : "Standard.LeftFoot", diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index fb42f89048..d697e1869a 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -780,18 +780,6 @@ void MyAvatar::update(float deltaTime) { emit energyChanged(currentEnergy); updateEyeContactTarget(deltaTime); - - // if we're getting eye rotations from a tracker, disable observer-side procedural eye motions - auto userInputMapper = DependencyManager::get(); - bool eyesTracked = - userInputMapper->getPoseState(controller::Action::LEFT_EYE).valid && - userInputMapper->getPoseState(controller::Action::RIGHT_EYE).valid; - - int leftEyeJointIndex = getJointIndex("LeftEye"); - int rightEyeJointIndex = getJointIndex("RightEye"); - bool eyesAreOverridden = getIsJointOverridden(leftEyeJointIndex) || getIsJointOverridden(rightEyeJointIndex); - - _headData->setHasProceduralEyeMovement(!(eyesTracked || eyesAreOverridden)); } void MyAvatar::updateEyeContactTarget(float deltaTime) { diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 73259fe073..f92f8b7218 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -21,6 +21,73 @@ using namespace std; +static controller::Action blendshapeActions[] = { + controller::Action::EYEBLINK_L, + controller::Action::EYEBLINK_R, + controller::Action::EYESQUINT_L, + controller::Action::EYESQUINT_R, + controller::Action::EYEDOWN_L, + controller::Action::EYEDOWN_R, + controller::Action::EYEIN_L, + controller::Action::EYEIN_R, + controller::Action::EYEOPEN_L, + controller::Action::EYEOPEN_R, + controller::Action::EYEOUT_L, + controller::Action::EYEOUT_R, + controller::Action::EYEUP_L, + controller::Action::EYEUP_R, + controller::Action::BROWSD_L, + controller::Action::BROWSD_R, + controller::Action::BROWSU_C, + controller::Action::BROWSU_L, + controller::Action::BROWSU_R, + controller::Action::JAWFWD, + controller::Action::JAWLEFT, + controller::Action::JAWOPEN, + controller::Action::JAWRIGHT, + controller::Action::MOUTHLEFT, + controller::Action::MOUTHRIGHT, + controller::Action::MOUTHFROWN_L, + controller::Action::MOUTHFROWN_R, + controller::Action::MOUTHSMILE_L, + controller::Action::MOUTHSMILE_R, + controller::Action::MOUTHDIMPLE_L, + controller::Action::MOUTHDIMPLE_R, + controller::Action::LIPSSTRETCH_L, + controller::Action::LIPSSTRETCH_R, + controller::Action::LIPSUPPERCLOSE, + controller::Action::LIPSLOWERCLOSE, + controller::Action::LIPSUPPEROPEN, + controller::Action::LIPSLOWEROPEN, + controller::Action::LIPSFUNNEL, + controller::Action::LIPSPUCKER, + controller::Action::PUFF, + controller::Action::CHEEKSQUINT_L, + controller::Action::CHEEKSQUINT_R, + controller::Action::LIPSTOGETHER, + controller::Action::MOUTHUPPERUP_L, + controller::Action::MOUTHUPPERUP_R, + controller::Action::MOUTHLOWERDOWN_L, + controller::Action::MOUTHLOWERDOWN_R, + controller::Action::MOUTHPRESS_L, + controller::Action::MOUTHPRESS_R, + controller::Action::MOUTHSHRUGLOWER, + controller::Action::MOUTHSHRUGUPPER, + controller::Action::NOSESNEER_L, + controller::Action::NOSESNEER_R, + controller::Action::TONGUEOUT, + controller::Action::USERBLENDSHAPE0, + controller::Action::USERBLENDSHAPE1, + controller::Action::USERBLENDSHAPE2, + controller::Action::USERBLENDSHAPE3, + controller::Action::USERBLENDSHAPE4, + controller::Action::USERBLENDSHAPE5, + controller::Action::USERBLENDSHAPE6, + controller::Action::USERBLENDSHAPE7, + controller::Action::USERBLENDSHAPE8, + controller::Action::USERBLENDSHAPE9 +}; + MyHead::MyHead(MyAvatar* owningAvatar) : Head(owningAvatar) { } @@ -44,40 +111,57 @@ void MyHead::simulate(float deltaTime) { auto player = DependencyManager::get(); // Only use face trackers when not playing back a recording. if (!player->isPlaying()) { - // TODO -- finish removing face-tracker specific code. To do this, add input channels for - // each blendshape-coefficient and update the various json files to relay them in a useful way. - // After that, input plugins can be used to drive the avatar's face, and the various "DDE" files - // can be ported into the plugin and removed. - // - // auto faceTracker = qApp->getActiveFaceTracker(); - // const bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); - // _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); - // if (_isFaceTrackerConnected) { - // if (hasActualFaceTrackerConnected) { - // _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); - // } - // } - - // AJT: blendshapes auto userInputMapper = DependencyManager::get(); + + // if input system has control over blink blendshapes bool eyeLidsTracked = - userInputMapper->getActionStateValid(controller::Action::LEFT_EYE_BLINK) && - userInputMapper->getActionStateValid(controller::Action::RIGHT_EYE_BLINK); + userInputMapper->getActionStateValid(controller::Action::EYEBLINK_L) || + userInputMapper->getActionStateValid(controller::Action::EYEBLINK_R); - setHasScriptedBlendshapes(eyeLidsTracked); + // if input system has control over the brows. + bool browsTracked = + userInputMapper->getActionStateValid(controller::Action::BROWSD_L) || + userInputMapper->getActionStateValid(controller::Action::BROWSD_R) || + userInputMapper->getActionStateValid(controller::Action::BROWSU_L) || + userInputMapper->getActionStateValid(controller::Action::BROWSU_R) || + userInputMapper->getActionStateValid(controller::Action::BROWSU_C); - if (eyeLidsTracked) { - float leftEyeBlink = userInputMapper->getActionState(controller::Action::LEFT_EYE_BLINK); - float rightEyeBlink = userInputMapper->getActionState(controller::Action::RIGHT_EYE_BLINK); - _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = leftEyeBlink; - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = rightEyeBlink; - } else { - const float FULLY_OPEN = 0.0f; - _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = FULLY_OPEN; - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = FULLY_OPEN; + // if input system has control of mouth + bool mouthTracked = + userInputMapper->getActionStateValid(controller::Action::JAWOPEN) || + userInputMapper->getActionStateValid(controller::Action::LIPSUPPERCLOSE) || + userInputMapper->getActionStateValid(controller::Action::LIPSLOWERCLOSE) || + userInputMapper->getActionStateValid(controller::Action::LIPSFUNNEL) || + userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_L) || + userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_R); + + bool eyesTracked = + userInputMapper->getPoseState(controller::Action::LEFT_EYE).valid && + userInputMapper->getPoseState(controller::Action::RIGHT_EYE).valid; + + MyAvatar* myAvatar = static_cast(_owningAvatar); + int leftEyeJointIndex = myAvatar->getJointIndex("LeftEye"); + int rightEyeJointIndex = myAvatar->getJointIndex("RightEye"); + bool eyeJointsOverridden = myAvatar->getIsJointOverridden(leftEyeJointIndex) || myAvatar->getIsJointOverridden(rightEyeJointIndex); + + bool anyInputTracked = false; + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { + anyInputTracked = anyInputTracked || userInputMapper->getActionStateValid(blendshapeActions[i]); + } + + setHasInputDrivenBlendshapes(anyInputTracked); + + // suppress any procedural blendshape animation if they overlap with driven input. + setSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, eyeLidsTracked); + setSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, eyeLidsTracked || browsTracked); + setSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, mouthTracked); + setSuppressProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation, eyesTracked || eyeJointsOverridden); + + if (anyInputTracked) { + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { + _blendshapeCoefficients[i] = userInputMapper->getActionState(blendshapeActions[i]); + } } } Parent::simulate(deltaTime); diff --git a/interface/src/avatar/MySkeletonModel.cpp b/interface/src/avatar/MySkeletonModel.cpp index 8d92767321..0d7bd3f3b3 100755 --- a/interface/src/avatar/MySkeletonModel.cpp +++ b/interface/src/avatar/MySkeletonModel.cpp @@ -114,6 +114,9 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { Head* head = _owningAvatar->getHead(); + // AJT: blendshapes TODO: RE-enable this and avoid duplication with + // SkeletonModel::updateRig() + /* bool eyePosesValid = !head->getHasProceduralEyeMovement(); glm::vec3 lookAt; if (eyePosesValid) { @@ -121,6 +124,8 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { } else { lookAt = avoidCrossedEyes(head->getLookAtPosition()); } + */ + glm::vec3 lookAt = avoidCrossedEyes(head->getLookAtPosition()); MyAvatar* myAvatar = static_cast(_owningAvatar); assert(myAvatar); diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index c7f807509c..2530fea4e7 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -72,7 +72,8 @@ void Head::simulate(float deltaTime) { _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); } - if (getHasProceduralEyeMovement()) { + if (getProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation)) { // Update eye saccades const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; const float AVERAGE_SACCADE_INTERVAL = 6.0f; @@ -95,7 +96,8 @@ void Head::simulate(float deltaTime) { const float BLINK_START_VARIABILITY = 0.25f; const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; - if (getHasProceduralBlinkFaceMovement()) { + if (getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation)) { // handle automatic blinks // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; @@ -146,7 +148,8 @@ void Head::simulate(float deltaTime) { } // use data to update fake Faceshift blendshape coefficients - if (getHasAudioEnabledFaceMovement()) { + if (getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) { // Update audio attack data for facial animation (eyebrows and mouth) float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz _audioAttack = audioAttackAveragingRate * _audioAttack + @@ -178,7 +181,8 @@ void Head::simulate(float deltaTime) { _mouth4, _transientBlendshapeCoefficients); - if (getHasProceduralEyeFaceMovement()) { + if (getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation)) { // This controls two things, the eye brow and the upper eye lid, it is driven by the vertical up/down angle of the // eyes relative to the head. This is to try to help prevent sleepy eyes/crazy eyes. applyEyelidOffset(getOrientation()); diff --git a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp index e0fed08955..b52a68f066 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp @@ -111,6 +111,8 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { Head* head = _owningAvatar->getHead(); + // AJT: blendshapes TODO: RE-enable this. but move into rig? + /* bool eyePosesValid = !head->getHasProceduralEyeMovement(); glm::vec3 lookAt; if (eyePosesValid) { @@ -118,6 +120,8 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { } else { lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); } + */ + glm::vec3 lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); // no need to call Model::updateRig() because otherAvatars get their joint state // copied directly from AvtarData::_jointData (there are no Rig animations to blend) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 520241d020..c5dcfba1c0 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -110,7 +110,6 @@ AvatarData::AvatarData() : _targetScale(1.0f), _handState(0), _keyState(NO_KEY_DOWN), - _hasScriptedBlendshapes(false), _headData(NULL), _errorLogExpiry(0), _owningAvatarMixer(), @@ -156,7 +155,7 @@ float AvatarData::getDomainLimitedScale() const { void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { - if (hasScriptedBlendshapes == _hasScriptedBlendshapes) { + if (hasScriptedBlendshapes == _headData->getHasScriptedBlendshapes()) { return; } if (!hasScriptedBlendshapes) { @@ -165,19 +164,35 @@ void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { // before sending the update, or else it won't send the neutal blendshapes to the receiving clients sendAvatarDataPacket(true); } - _hasScriptedBlendshapes = hasScriptedBlendshapes; + _headData->setHasScriptedBlendshapes(hasScriptedBlendshapes); } -void AvatarData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); +bool AvatarData::getHasScriptedBlendshapes() const { + return _headData->getHasScriptedBlendshapes(); } -void AvatarData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); +void AvatarData::setHasProceduralBlinkFaceMovement(bool value) { + _headData->setProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, value); } -void AvatarData::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); +bool AvatarData::getHasProceduralBlinkFaceMovement() const { + return _headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation); +} + +void AvatarData::setHasProceduralEyeFaceMovement(bool value) { + _headData->setProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, value); +} + +bool AvatarData::getHasProceduralEyeFaceMovement() const { + return _headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation); +} + +void AvatarData::setHasAudioEnabledFaceMovement(bool value) { + _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, value); +} + +bool AvatarData::getHasAudioEnabledFaceMovement() const { + return _headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation); } void AvatarData::setDomainMinimumHeight(float domainMinimumHeight) { @@ -232,9 +247,6 @@ void AvatarData::lazyInitHeadData() const { if (!_headData) { _headData = new HeadData(const_cast(this)); } - if (_hasScriptedBlendshapes) { - _headData->_hasScriptedBlendshapes = true; - } } @@ -555,27 +567,31 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent setAtBit16(flags, HAND_STATE_FINGER_POINTING_BIT); } // face tracker state - if (_headData->_hasScriptedBlendshapes) { + if (_headData->_hasScriptedBlendshapes || _headData->_hasInputDrivenBlendshapes) { setAtBit16(flags, HAS_SCRIPTED_BLENDSHAPES); } // eye tracker state - if (!_headData->_hasProceduralEyeMovement) { - setAtBit16(flags, IS_EYE_TRACKER_CONNECTED); + if (_headData->getProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation)) { + setAtBit16(flags, HAS_PROCEDURAL_EYE_MOVEMENT); } // referential state if (!parentID.isNull()) { setAtBit16(flags, HAS_REFERENTIAL); } // audio face movement - if (_headData->getHasAudioEnabledFaceMovement()) { + if (_headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) { setAtBit16(flags, AUDIO_ENABLED_FACE_MOVEMENT); } // procedural eye face movement - if (_headData->getHasProceduralEyeFaceMovement()) { + if (_headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation)) { setAtBit16(flags, PROCEDURAL_EYE_FACE_MOVEMENT); } // procedural blink face movement - if (_headData->getHasProceduralBlinkFaceMovement()) { + if (_headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation)) { setAtBit16(flags, PROCEDURAL_BLINK_FACE_MOVEMENT); } // avatar collisions enabled @@ -1177,21 +1193,22 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { + (oneAtBit16(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0); auto newHasScriptedBlendshapes = oneAtBit16(bitItems, HAS_SCRIPTED_BLENDSHAPES); - auto newHasntProceduralEyeMovement = oneAtBit16(bitItems, IS_EYE_TRACKER_CONNECTED); - + auto newHasProceduralEyeMovement = oneAtBit16(bitItems, HAS_PROCEDURAL_EYE_MOVEMENT); auto newHasAudioEnabledFaceMovement = oneAtBit16(bitItems, AUDIO_ENABLED_FACE_MOVEMENT); auto newHasProceduralEyeFaceMovement = oneAtBit16(bitItems, PROCEDURAL_EYE_FACE_MOVEMENT); auto newHasProceduralBlinkFaceMovement = oneAtBit16(bitItems, PROCEDURAL_BLINK_FACE_MOVEMENT); + auto newCollideWithOtherAvatars = oneAtBit16(bitItems, COLLIDE_WITH_OTHER_AVATARS); auto newHasPriority = oneAtBit16(bitItems, HAS_HERO_PRIORITY); bool keyStateChanged = (_keyState != newKeyState); bool handStateChanged = (_handState != newHandState); - bool faceStateChanged = (_headData->_hasScriptedBlendshapes != newHasScriptedBlendshapes); - bool eyeStateChanged = (_headData->_hasProceduralEyeMovement == newHasntProceduralEyeMovement); - bool audioEnableFaceMovementChanged = (_headData->getHasAudioEnabledFaceMovement() != newHasAudioEnabledFaceMovement); - bool proceduralEyeFaceMovementChanged = (_headData->getHasProceduralEyeFaceMovement() != newHasProceduralEyeFaceMovement); - bool proceduralBlinkFaceMovementChanged = (_headData->getHasProceduralBlinkFaceMovement() != newHasProceduralBlinkFaceMovement); + bool faceStateChanged = (_headData->getHasScriptedBlendshapes() != newHasScriptedBlendshapes); + + bool eyeStateChanged = (_headData->getProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation) != newHasProceduralEyeMovement); + bool audioEnableFaceMovementChanged = (_headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) != newHasAudioEnabledFaceMovement); + bool proceduralEyeFaceMovementChanged = (_headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) != newHasProceduralEyeFaceMovement); + bool proceduralBlinkFaceMovementChanged = (_headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) != newHasProceduralBlinkFaceMovement); bool collideWithOtherAvatarsChanged = (_collideWithOtherAvatars != newCollideWithOtherAvatars); bool hasPriorityChanged = (getHasPriority() != newHasPriority); bool somethingChanged = keyStateChanged || handStateChanged || faceStateChanged || eyeStateChanged || audioEnableFaceMovementChanged || @@ -1200,11 +1217,11 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _keyState = newKeyState; _handState = newHandState; - _headData->_hasScriptedBlendshapes = newHasScriptedBlendshapes; - _headData->setHasProceduralEyeMovement(!newHasntProceduralEyeMovement); - _headData->setHasAudioEnabledFaceMovement(newHasAudioEnabledFaceMovement); - _headData->setHasProceduralEyeFaceMovement(newHasProceduralEyeFaceMovement); - _headData->setHasProceduralBlinkFaceMovement(newHasProceduralBlinkFaceMovement); + _headData->setHasScriptedBlendshapes(newHasScriptedBlendshapes); + _headData->setProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation, newHasProceduralEyeMovement); + _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, newHasAudioEnabledFaceMovement); + _headData->setProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, newHasProceduralEyeFaceMovement); + _headData->setProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, newHasProceduralBlinkFaceMovement); _collideWithOtherAvatars = newCollideWithOtherAvatars; setHasPriorityWithoutTimestampReset(newHasPriority); @@ -1289,7 +1306,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { sourceBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); PACKET_READ_CHECK(FaceTrackerCoefficients, coefficientsSize); - _headData->_blendshapeCoefficients.resize(numCoefficients); // make sure there's room for the copy! + _headData->_blendshapeCoefficients.resize(std::min(numCoefficients, (int)Blendshapes::BlendshapeCount)); // make sure there's room for the copy! //only copy the blendshapes to headData, not the procedural face info memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, coefficientsSize); sourceBuffer += coefficientsSize; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 6bed5fa9db..53903412d0 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -107,7 +107,7 @@ const quint32 AVATAR_MOTION_SCRIPTABLE_BITS = const int KEY_STATE_START_BIT = 0; // 1st and 2nd bits (UNUSED) const int HAND_STATE_START_BIT = 2; // 3rd and 4th bits (UNUSED) const int HAS_SCRIPTED_BLENDSHAPES = 4; // 5th bit -const int IS_EYE_TRACKER_CONNECTED = 5; // 6th bit (was CHAT_CIRCLING) +const int HAS_PROCEDURAL_EYE_MOVEMENT = 5; // 6th bit const int HAS_REFERENTIAL = 6; // 7th bit const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit (UNUSED) const int AUDIO_ENABLED_FACE_MOVEMENT = 8; // 9th bit @@ -703,13 +703,13 @@ public: float getDomainLimitedScale() const; void setHasScriptedBlendshapes(bool hasScriptedBlendshapes); - bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } + bool getHasScriptedBlendshapes() const; void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasProceduralBlinkFaceMovement() const { return _headData->getHasProceduralBlinkFaceMovement(); } + bool getHasProceduralBlinkFaceMovement() const; void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralEyeFaceMovement() const { return _headData->getHasProceduralEyeFaceMovement(); } + bool getHasProceduralEyeFaceMovement() const; void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasAudioEnabledFaceMovement() const { return _headData->getHasAudioEnabledFaceMovement(); } + bool getHasAudioEnabledFaceMovement() const; /**jsdoc * Gets the minimum scale allowed for this avatar in the current domain. @@ -1716,7 +1716,6 @@ protected: // key state KeyState _keyState; - bool _hasScriptedBlendshapes; bool _hasNewJointData { true }; // set in AvatarData, cleared in Avatar mutable HeadData* _headData { nullptr }; diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 561f2c798a..f37b88c135 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -27,11 +27,10 @@ HeadData::HeadData(AvatarData* owningAvatar) : _basePitch(0.0f), _baseRoll(0.0f), _lookAtPosition(0.0f, 0.0f, 0.0f), - _blendshapeCoefficients(QVector(0, 0.0f)), - _transientBlendshapeCoefficients(QVector(0, 0.0f)), - _summedBlendshapeCoefficients(QVector(0, 0.0f)), _owningAvatar(owningAvatar) { + _userProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, true); + _suppressProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, false); computeBlendshapesLookupMap(); } @@ -102,7 +101,7 @@ const QVector& HeadData::getSummedBlendshapeCoefficients() { void HeadData::setBlendshape(QString name, float val) { - //Check to see if the named blendshape exists, and then set its value if it does + // Check to see if the named blendshape exists, and then set its value if it does auto it = _blendshapeLookupMap.find(name); if (it != _blendshapeLookupMap.end()) { if (_blendshapeCoefficients.size() <= it.value()) { @@ -112,6 +111,19 @@ void HeadData::setBlendshape(QString name, float val) { _transientBlendshapeCoefficients.resize(it.value() + 1); } _blendshapeCoefficients[it.value()] = val; + } else { + // check to see if this is a legacy blendshape that is present in + // ARKit blendshapes but is split. i.e. has left and right halfs. + if (name == "LipsUpperUp") { + _blendshapeCoefficients[(int)Blendshapes::MouthUpperUp_L] = val; + _blendshapeCoefficients[(int)Blendshapes::MouthUpperUp_R] = val; + } else if (name == "LipsLowerDown") { + _blendshapeCoefficients[(int)Blendshapes::MouthLowerDown_L] = val; + _blendshapeCoefficients[(int)Blendshapes::MouthLowerDown_R] = val; + } else if (name == "Sneer") { + _blendshapeCoefficients[(int)Blendshapes::NoseSneer_L] = val; + _blendshapeCoefficients[(int)Blendshapes::NoseSneer_R] = val; + } } } @@ -197,38 +209,34 @@ void HeadData::fromJson(const QJsonObject& json) { } } -bool HeadData::getHasProceduralEyeFaceMovement() const { - return _hasProceduralEyeFaceMovement; +bool HeadData::getProceduralAnimationFlag(ProceduralAnimationType type) const { + return _userProceduralAnimationFlags[(int)type]; } -void HeadData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _hasProceduralEyeFaceMovement = hasProceduralEyeFaceMovement; +void HeadData::setProceduralAnimationFlag(ProceduralAnimationType type, bool value) { + _userProceduralAnimationFlags[(int)type] = value; } -bool HeadData::getHasProceduralBlinkFaceMovement() const { - return _hasProceduralBlinkFaceMovement; +bool HeadData::getSuppressProceduralAnimationFlag(ProceduralAnimationType type) const { + return _suppressProceduralAnimationFlags[(int)type]; } -void HeadData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _hasProceduralBlinkFaceMovement = hasProceduralBlinkFaceMovement; +void HeadData::setSuppressProceduralAnimationFlag(ProceduralAnimationType type, bool value) { + _suppressProceduralAnimationFlags[(int)type] = value; } -bool HeadData::getHasAudioEnabledFaceMovement() const { - return _hasAudioEnabledFaceMovement; -} - -void HeadData::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _hasAudioEnabledFaceMovement = hasAudioEnabledFaceMovement; -} - -bool HeadData::getHasProceduralEyeMovement() const { - return _hasProceduralEyeMovement; -} - -void HeadData::setHasProceduralEyeMovement(bool hasProceduralEyeMovement) { - _hasProceduralEyeMovement = hasProceduralEyeMovement; +bool HeadData::getHasScriptedBlendshapes() const { + return _hasScriptedBlendshapes; } void HeadData::setHasScriptedBlendshapes(bool value) { _hasScriptedBlendshapes = value; } + +bool HeadData::getHasInputDrivenBlendshapes() const { + return _hasInputDrivenBlendshapes; +} + +void HeadData::setHasInputDrivenBlendshapes(bool value) { + _hasInputDrivenBlendshapes = value; +} diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index c6bcc2599b..9652792512 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -72,17 +72,29 @@ public: } bool lookAtPositionChangedSince(quint64 time) { return _lookAtPositionChanged >= time; } - bool getHasProceduralEyeFaceMovement() const; - void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralBlinkFaceMovement() const; - void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasAudioEnabledFaceMovement() const; - void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasProceduralEyeMovement() const; - void setHasProceduralEyeMovement(bool hasProceduralEyeMovement); + enum ProceduralAnimationType { + AudioProceduralBlendshapeAnimation = 0, + BlinkProceduralBlendshapeAnimation, + LidAdjustmentProceduralBlendshapeAnimation, + SaccadeProceduralEyeJointAnimation, + ProceduralAnimaitonTypeCount, + }; + // called by scripts to enable or disable procedural blendshape or eye joint animations. + bool getProceduralAnimationFlag(ProceduralAnimationType type) const; + void setProceduralAnimationFlag(ProceduralAnimationType type, bool value); + + // called by c++ to suppress, i.e. temporarily disable a procedural animation. + bool getSuppressProceduralAnimationFlag(ProceduralAnimationType flag) const; + void setSuppressProceduralAnimationFlag(ProceduralAnimationType flag, bool value); + + // called by scripts to enable/disable manual adjustment of blendshapes void setHasScriptedBlendshapes(bool value); - bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } + bool getHasScriptedBlendshapes() const; + + // called by C++ code to denote the presence of manually driven blendshapes. + void setHasInputDrivenBlendshapes(bool value); + bool getHasInputDrivenBlendshapes() const; friend class AvatarData; @@ -98,21 +110,20 @@ protected: glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; - bool _hasAudioEnabledFaceMovement { true }; - bool _hasProceduralBlinkFaceMovement { true }; - bool _hasProceduralEyeFaceMovement { true }; - bool _hasProceduralEyeMovement { true }; + std::vector _userProceduralAnimationFlags; + std::vector _suppressProceduralAnimationFlags; bool _hasScriptedBlendshapes { false }; + bool _hasInputDrivenBlendshapes { false }; float _leftEyeBlink { 0.0f }; float _rightEyeBlink { 0.0f }; float _averageLoudness { 0.0f }; float _browAudioLift { 0.0f }; - QVector _blendshapeCoefficients; - QVector _transientBlendshapeCoefficients; - QVector _summedBlendshapeCoefficients; + QVector _blendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; + QVector _transientBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; + QVector _summedBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; QMap _blendshapeLookupMap; AvatarData* _owningAvatar; diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index e1ad17aafa..1a5c73a1d8 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -185,70 +185,70 @@ enum class Action { RIGHT_EYE, // AJT: blendshapes - EyeBlink_L, - EyeBlink_R, - EyeSquint_L, - EyeSquint_R, - EyeDown_L, - EyeDown_R, - EyeIn_L, - EyeIn_R, - EyeOpen_L, - EyeOpen_R, - EyeOut_L, - EyeOut_R, - EyeUp_L, - EyeUp_R, - BrowsD_L, - BrowsD_R, - BrowsU_C, - BrowsU_L, - BrowsU_R, - JawFwd, - JawLeft, - JawOpen, - JawRight, - MouthLeft, - MouthRight, - MouthFrown_L, - MouthFrown_R, - MouthSmile_L, - MouthSmile_R, - MouthDimple_L, - MouthDimple_R, - LipsStretch_L, - LipsStretch_R, - LipsUpperClose, - LipsLowerClose, - LipsUpperOpen, - LipsLowerOpen, - LipsFunnel, - LipsPucker, - Puff, - CheekSquint_L, - CheekSquint_R, - LipsTogether, - MouthUpperUp_L, - MouthUpperUp_R, - MouthLowerDown_L, - MouthLowerDown_R, - MouthPress_L, - MouthPress_R, - MouthShrugLower, - MouthShrugUpper, - NoseSneer_L, - NoseSneer_R, - TongueOut, - UserBlendshape0, - UserBlendshape1, - UserBlendshape2, - UserBlendshape3, - UserBlendshape4, - UserBlendshape5, - UserBlendshape6, - UserBlendshape7, - UserBlendshape8, - UserBlendshape9, + EYEBLINK_L, + EYEBLINK_R, + EYESQUINT_L, + EYESQUINT_R, + EYEDOWN_L, + EYEDOWN_R, + EYEIN_L, + EYEIN_R, + EYEOPEN_L, + EYEOPEN_R, + EYEOUT_L, + EYEOUT_R, + EYEUP_L, + EYEUP_R, + BROWSD_L, + BROWSD_R, + BROWSU_C, + BROWSU_L, + BROWSU_R, + JAWFWD, + JAWLEFT, + JAWOPEN, + JAWRIGHT, + MOUTHLEFT, + MOUTHRIGHT, + MOUTHFROWN_L, + MOUTHFROWN_R, + MOUTHSMILE_L, + MOUTHSMILE_R, + MOUTHDIMPLE_L, + MOUTHDIMPLE_R, + LIPSSTRETCH_L, + LIPSSTRETCH_R, + LIPSUPPERCLOSE, + LIPSLOWERCLOSE, + LIPSUPPEROPEN, + LIPSLOWEROPEN, + LIPSFUNNEL, + LIPSPUCKER, + PUFF, + CHEEKSQUINT_L, + CHEEKSQUINT_R, + LIPSTOGETHER, + MOUTHUPPERUP_L, + MOUTHUPPERUP_R, + MOUTHLOWERDOWN_L, + MOUTHLOWERDOWN_R, + MOUTHPRESS_L, + MOUTHPRESS_R, + MOUTHSHRUGLOWER, + MOUTHSHRUGUPPER, + NOSESNEER_L, + NOSESNEER_R, + TONGUEOUT, + USERBLENDSHAPE0, + USERBLENDSHAPE1, + USERBLENDSHAPE2, + USERBLENDSHAPE3, + USERBLENDSHAPE4, + USERBLENDSHAPE5, + USERBLENDSHAPE6, + USERBLENDSHAPE7, + USERBLENDSHAPE8, + USERBLENDSHAPE9, NUM_ACTIONS }; diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index 2af822ffc2..55bcf0e36a 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -92,8 +92,70 @@ namespace controller { RIGHT_GRIP, // AJT: blendshapes - LEFT_EYE_BLINK, - RIGHT_EYE_BLINK, + EYEBLINK_L, + EYEBLINK_R, + EYESQUINT_L, + EYESQUINT_R, + EYEDOWN_L, + EYEDOWN_R, + EYEIN_L, + EYEIN_R, + EYEOPEN_L, + EYEOPEN_R, + EYEOUT_L, + EYEOUT_R, + EYEUP_L, + EYEUP_R, + BROWSD_L, + BROWSD_R, + BROWSU_C, + BROWSU_L, + BROWSU_R, + JAWFWD, + JAWLEFT, + JAWOPEN, + JAWRIGHT, + MOUTHLEFT, + MOUTHRIGHT, + MOUTHFROWN_L, + MOUTHFROWN_R, + MOUTHSMILE_L, + MOUTHSMILE_R, + MOUTHDIMPLE_L, + MOUTHDIMPLE_R, + LIPSSTRETCH_L, + LIPSSTRETCH_R, + LIPSUPPERCLOSE, + LIPSLOWERCLOSE, + LIPSUPPEROPEN, + LIPSLOWEROPEN, + LIPSFUNNEL, + LIPSPUCKER, + PUFF, + CHEEKSQUINT_L, + CHEEKSQUINT_R, + LIPSTOGETHER, + MOUTHUPPERUP_L, + MOUTHUPPERUP_R, + MOUTHLOWERDOWN_L, + MOUTHLOWERDOWN_R, + MOUTHPRESS_L, + MOUTHPRESS_R, + MOUTHSHRUGLOWER, + MOUTHSHRUGUPPER, + NOSESNEER_L, + NOSESNEER_R, + TONGUEOUT, + USERBLENDSHAPE0, + USERBLENDSHAPE1, + USERBLENDSHAPE2, + USERBLENDSHAPE3, + USERBLENDSHAPE4, + USERBLENDSHAPE5, + USERBLENDSHAPE6, + USERBLENDSHAPE7, + USERBLENDSHAPE8, + USERBLENDSHAPE9, NUM_STANDARD_AXES, LZ = LT, diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index ed68fe89dc..87bd7941d3 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -38,10 +38,10 @@ PacketVersion versionForPacketType(PacketType packetType) { return static_cast(EntityQueryPacketVersion::ConicalFrustums); case PacketType::AvatarIdentity: case PacketType::AvatarData: - return static_cast(AvatarMixerPacketVersion::SendVerificationFailed); + return static_cast(AvatarMixerPacketVersion::ARKitBlendshapes); case PacketType::BulkAvatarData: case PacketType::KillAvatar: - return static_cast(AvatarMixerPacketVersion::SendVerificationFailed); + return static_cast(AvatarMixerPacketVersion::ARKitBlendshapes); case PacketType::MessagesData: return static_cast(MessageDataVersion::TextOrBinaryData); // ICE packets diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 52a8c8db16..fbf575065e 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -339,7 +339,8 @@ enum class AvatarMixerPacketVersion : PacketVersion { SendMaxTranslationDimension, FBXJointOrderChange, HandControllerSection, - SendVerificationFailed + SendVerificationFailed, + ARKitBlendshapes }; enum class DomainConnectRequestVersion : PacketVersion { diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 97b459fa0e..63e99f465f 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -91,7 +91,7 @@ enum class LegacyBlendshpaes : int { ChinUpperRaise, // not in ARKit Sneer, // split in ARKit LegacyBlendshapeCount -} +}; // NEW in ARKit // * LipsTogether From 92f1b59cc8e555e0594cad64eb5151a3378f888e Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 18 Oct 2019 16:30:32 -0700 Subject: [PATCH 05/19] Full legacy API and FST support --- .../src/avatars/ScriptableAvatar.cpp | 12 ----- .../src/avatars/ScriptableAvatar.h | 7 --- libraries/avatars/src/HeadData.cpp | 3 ++ libraries/avatars/src/HeadData.h | 6 +-- .../controllers/src/controllers/Actions.cpp | 2 +- .../controllers/src/controllers/Actions.h | 2 +- .../src/controllers/StandardController.cpp | 2 +- .../src/controllers/StandardControls.h | 2 +- libraries/fbx/src/FSTReader.cpp | 50 +++++++++++++++++-- libraries/render-utils/src/Model.cpp | 2 + libraries/shared/src/BlendshapeConstants.h | 6 +-- 11 files changed, 60 insertions(+), 34 deletions(-) diff --git a/assignment-client/src/avatars/ScriptableAvatar.cpp b/assignment-client/src/avatars/ScriptableAvatar.cpp index 044ab86942..54eb499be1 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.cpp +++ b/assignment-client/src/avatars/ScriptableAvatar.cpp @@ -279,18 +279,6 @@ void ScriptableAvatar::setJointMappingsFromNetworkReply() { networkReply->deleteLater(); } -void ScriptableAvatar::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); -} - -void ScriptableAvatar::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); -} - -void ScriptableAvatar::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); -} - AvatarEntityMap ScriptableAvatar::getAvatarEntityData() const { // DANGER: Now that we store the AvatarEntityData in packed format this call is potentially Very Expensive! // Avoid calling this method if possible. diff --git a/assignment-client/src/avatars/ScriptableAvatar.h b/assignment-client/src/avatars/ScriptableAvatar.h index fc796b418f..f2f5a1e6f4 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.h +++ b/assignment-client/src/avatars/ScriptableAvatar.h @@ -153,13 +153,6 @@ public: virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking = false) override; - void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasProceduralBlinkFaceMovement() const override { return _headData->getHasProceduralBlinkFaceMovement(); } - void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralEyeFaceMovement() const override { return _headData->getHasProceduralEyeFaceMovement(); } - void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasAudioEnabledFaceMovement() const override { return _headData->getHasAudioEnabledFaceMovement(); } - /**jsdoc * Gets details of all avatar entities. *

Warning: Potentially an expensive call. Do not use if possible.

diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index f37b88c135..ff73677bb8 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -27,6 +27,9 @@ HeadData::HeadData(AvatarData* owningAvatar) : _basePitch(0.0f), _baseRoll(0.0f), _lookAtPosition(0.0f, 0.0f, 0.0f), + _blendshapeCoefficients((int)Blendshapes::BlendshapeCount, 0.0f), + _transientBlendshapeCoefficients((int)Blendshapes::BlendshapeCount, 0.0f), + _summedBlendshapeCoefficients((int)Blendshapes::BlendshapeCount, 0.0f), _owningAvatar(owningAvatar) { _userProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, true); diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 9652792512..218ffceaa1 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -121,9 +121,9 @@ protected: float _averageLoudness { 0.0f }; float _browAudioLift { 0.0f }; - QVector _blendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; - QVector _transientBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; - QVector _summedBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; + QVector _blendshapeCoefficients; + QVector _transientBlendshapeCoefficients; + QVector _summedBlendshapeCoefficients; QMap _blendshapeLookupMap; AvatarData* _owningAvatar; diff --git a/libraries/controllers/src/controllers/Actions.cpp b/libraries/controllers/src/controllers/Actions.cpp index 198c342b1d..6dd9a47395 100644 --- a/libraries/controllers/src/controllers/Actions.cpp +++ b/libraries/controllers/src/controllers/Actions.cpp @@ -350,7 +350,7 @@ namespace controller { makePosePair(Action::LEFT_EYE, "LeftEye"), makePosePair(Action::RIGHT_EYE, "RightEye"), - // AJT: blendshapes + // blendshapes makeAxisPair(Action::EYEBLINK_L, "EyeBlink_L"), makeAxisPair(Action::EYEBLINK_R, "EyeBlink_R"), makeAxisPair(Action::EYESQUINT_L, "EyeSquint_L"), diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index 1a5c73a1d8..86821b4633 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -184,7 +184,7 @@ enum class Action { LEFT_EYE, RIGHT_EYE, - // AJT: blendshapes + // blendshapes EYEBLINK_L, EYEBLINK_R, EYESQUINT_L, diff --git a/libraries/controllers/src/controllers/StandardController.cpp b/libraries/controllers/src/controllers/StandardController.cpp index 14697b26d2..168604ee45 100644 --- a/libraries/controllers/src/controllers/StandardController.cpp +++ b/libraries/controllers/src/controllers/StandardController.cpp @@ -356,7 +356,7 @@ Input::NamedVector StandardController::getAvailableInputs() const { makePair(LEFT_EYE, "LeftEye"), makePair(RIGHT_EYE, "RightEye"), - // AJT: blendshapes + // blendshapes makePair(EYEBLINK_L, "EyeBlink_L"), makePair(EYEBLINK_R, "EyeBlink_R"), makePair(EYESQUINT_L, "EyeSquint_L"), diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index 55bcf0e36a..f99072af7c 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -91,7 +91,7 @@ namespace controller { LEFT_GRIP, RIGHT_GRIP, - // AJT: blendshapes + // blendshapes EYEBLINK_L, EYEBLINK_R, EYESQUINT_L, diff --git a/libraries/fbx/src/FSTReader.cpp b/libraries/fbx/src/FSTReader.cpp index 41b660f722..2835151bfe 100644 --- a/libraries/fbx/src/FSTReader.cpp +++ b/libraries/fbx/src/FSTReader.cpp @@ -21,7 +21,7 @@ QVariantHash FSTReader::parseMapping(QIODevice* device) { QVariantHash properties; - + QByteArray line; while (!(line = device->readLine()).isEmpty()) { if ((line = line.trimmed()).startsWith('#')) { @@ -34,12 +34,10 @@ QVariantHash FSTReader::parseMapping(QIODevice* device) { QByteArray name = sections.at(0).trimmed(); if (sections.size() == 2) { properties.insertMulti(name, sections.at(1).trimmed()); - } else if (sections.size() == 3) { QVariantHash heading = properties.value(name).toHash(); heading.insertMulti(sections.at(1).trimmed(), sections.at(2).trimmed()); properties.insert(name, heading); - } else if (sections.size() >= 4) { QVariantHash heading = properties.value(name).toHash(); QVariantList contents; @@ -50,14 +48,56 @@ QVariantHash FSTReader::parseMapping(QIODevice* device) { properties.insert(name, heading); } } - + return properties; } +static void removeBlendshape(QVariantHash& bs, const QString& key) { + if (bs.contains(key)) { + bs.remove(key); + } +} + +static void splitBlendshapes(QVariantHash& bs, const QString& key, const QString& leftKey, const QString& rightKey) { + if (bs.contains(key) && !(bs.contains(leftKey) || bs.contains(rightKey))) { + // key has been split into leftKey and rightKey blendshapes + QVariantList origShapes = bs.values(key); + QVariantList halfShapes; + for (int i = 0; i < origShapes.size(); i++) { + QVariantList origShape = origShapes[i].toList(); + QVariantList halfShape; + halfShape.append(origShape[0]); + halfShape.append(QVariant(0.5f * origShape[1].toFloat())); + bs.insertMulti(leftKey, halfShape); + bs.insertMulti(rightKey, halfShape); + } + } +} + +// convert legacy blendshapes to arkit blendshapes +static void fixUpLegacyBlendshapes(QVariantHash& properties) { + QVariantHash bs = properties.value("bs").toHash(); + + // These blendshapes have no ARKit equivalent, so we remove them. + removeBlendshape(bs, "JawChew"); + removeBlendshape(bs, "ChinLowerRaise"); + removeBlendshape(bs, "ChinUpperRaise"); + + // These blendshapes are split in ARKit, we replace them with their left and right sides with a weight of 1/2. + splitBlendshapes(bs, "LipsUpperUp", "MouthUpperUp_L", "MouthUpperUp_R"); + splitBlendshapes(bs, "LipsLowerDown", "MouthLowerDown_L", "MouthLowerDown_R"); + splitBlendshapes(bs, "Sneer", "NoseSneer_L", "NoseSneer_R"); + + // re-insert new mutated bs hash into mapping properties. + properties.insert("bs", bs); +} + QVariantHash FSTReader::readMapping(const QByteArray& data) { QBuffer buffer(const_cast(&data)); buffer.open(QIODevice::ReadOnly); - return FSTReader::parseMapping(&buffer); + QVariantHash mapping = FSTReader::parseMapping(&buffer); + fixUpLegacyBlendshapes(mapping); + return mapping; } void FSTReader::writeVariant(QBuffer& buffer, QVariantHash::const_iterator& it) { diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 74cf1ffa39..a7e098e1b7 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -39,6 +39,8 @@ #include "RenderUtilsLogging.h" #include +#include + using namespace std; int nakedModelPointerTypeId = qRegisterMetaType(); diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 63e99f465f..0f934b2056 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -109,10 +109,10 @@ enum class LegacyBlendshpaes : int { // Legacy shapes // * JawChew (not in ARKit) -// * MouthUpperUp (split in ARKit) -// * MouthLowerDown (split in ARKit) +// * LipsUpperUp (split in ARKit) +// * LipsLowerDown (split in ARKit) // * Sneer (split in ARKit) // * ChinLowerRaise (not in ARKit) -// * ChinUpperRase (not in ARKit) +// * ChinUpperRaise (not in ARKit) #endif // hifi_BlendshapeConstants_h From 03f88f696d1c1d8d31ec5649157b9f887f944850 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Mon, 21 Oct 2019 19:37:57 -0700 Subject: [PATCH 06/19] Renamed LipsTogether to MouthClose and bug fixes. --- interface/resources/controllers/standard.json | 2 +- interface/resources/controllers/standard_nomovement.json | 2 +- interface/src/avatar/MyHead.cpp | 2 +- libraries/avatars/src/AvatarData.cpp | 4 ++++ libraries/avatars/src/HeadData.cpp | 4 ++++ libraries/avatars/src/HeadData.h | 1 + libraries/controllers/src/controllers/Actions.cpp | 2 +- libraries/controllers/src/controllers/Actions.h | 2 +- libraries/controllers/src/controllers/StandardController.cpp | 2 +- libraries/controllers/src/controllers/StandardControls.h | 2 +- libraries/shared/src/BlendshapeConstants.cpp | 2 +- libraries/shared/src/BlendshapeConstants.h | 4 ++-- 12 files changed, 19 insertions(+), 10 deletions(-) diff --git a/interface/resources/controllers/standard.json b/interface/resources/controllers/standard.json index 8c84039b86..b6cb4e3e27 100644 --- a/interface/resources/controllers/standard.json +++ b/interface/resources/controllers/standard.json @@ -208,7 +208,7 @@ { "from": "Standard.Puff", "to": "Actions.Puff" }, { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, - { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthClose", "to": "Actions.MouthClose" }, { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, diff --git a/interface/resources/controllers/standard_nomovement.json b/interface/resources/controllers/standard_nomovement.json index e311a5fa62..04c0d2f329 100644 --- a/interface/resources/controllers/standard_nomovement.json +++ b/interface/resources/controllers/standard_nomovement.json @@ -103,7 +103,7 @@ { "from": "Standard.Puff", "to": "Actions.Puff" }, { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, - { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthClose", "to": "Actions.MouthClose" }, { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index f92f8b7218..a0e70a3049 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -64,7 +64,7 @@ static controller::Action blendshapeActions[] = { controller::Action::PUFF, controller::Action::CHEEKSQUINT_L, controller::Action::CHEEKSQUINT_R, - controller::Action::LIPSTOGETHER, + controller::Action::MOUTHCLOSE, controller::Action::MOUTHUPPERUP_L, controller::Action::MOUTHUPPERUP_R, controller::Action::MOUTHLOWERDOWN_L, diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index c5dcfba1c0..87d6c57d59 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1217,6 +1217,10 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _keyState = newKeyState; _handState = newHandState; + if (!newHasScriptedBlendshapes && getHasScriptedBlendshapes()) { + // if scripted blendshapes have just been turned off, slam blendshapes back to zero. + _headData->clearBlendshapeCoefficients(); + } _headData->setHasScriptedBlendshapes(newHasScriptedBlendshapes); _headData->setProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation, newHasProceduralEyeMovement); _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, newHasAudioEnabledFaceMovement); diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index ff73677bb8..93fb0190bb 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -83,6 +83,10 @@ int HeadData::getNumSummedBlendshapeCoefficients() const { return maxSize; } +void HeadData::clearBlendshapeCoefficients() { + _blendshapeCoefficients.fill(0.0f, (int)_blendshapeCoefficients.size()); +} + const QVector& HeadData::getSummedBlendshapeCoefficients() { int maxSize = std::max(_blendshapeCoefficients.size(), _transientBlendshapeCoefficients.size()); if (_summedBlendshapeCoefficients.size() != maxSize) { diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 218ffceaa1..083374b32d 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -62,6 +62,7 @@ public: const QVector& getSummedBlendshapeCoefficients(); int getNumSummedBlendshapeCoefficients() const; void setBlendshapeCoefficients(const QVector& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; } + void clearBlendshapeCoefficients(); const glm::vec3& getLookAtPosition() const { return _lookAtPosition; } void setLookAtPosition(const glm::vec3& lookAtPosition) { diff --git a/libraries/controllers/src/controllers/Actions.cpp b/libraries/controllers/src/controllers/Actions.cpp index 6dd9a47395..36f454b5d0 100644 --- a/libraries/controllers/src/controllers/Actions.cpp +++ b/libraries/controllers/src/controllers/Actions.cpp @@ -393,7 +393,7 @@ namespace controller { makeAxisPair(Action::PUFF, "Puff"), makeAxisPair(Action::CHEEKSQUINT_L, "CheekSquint_L"), makeAxisPair(Action::CHEEKSQUINT_R, "CheekSquint_R"), - makeAxisPair(Action::LIPSTOGETHER, "LipsTogether"), + makeAxisPair(Action::MOUTHCLOSE, "MouthClose"), makeAxisPair(Action::MOUTHUPPERUP_L, "MouthUpperUp_L"), makeAxisPair(Action::MOUTHUPPERUP_R, "MouthUpperUp_R"), makeAxisPair(Action::MOUTHLOWERDOWN_L, "MouthLowerDown_L"), diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index 86821b4633..5c96923dc3 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -227,7 +227,7 @@ enum class Action { PUFF, CHEEKSQUINT_L, CHEEKSQUINT_R, - LIPSTOGETHER, + MOUTHCLOSE, MOUTHUPPERUP_L, MOUTHUPPERUP_R, MOUTHLOWERDOWN_L, diff --git a/libraries/controllers/src/controllers/StandardController.cpp b/libraries/controllers/src/controllers/StandardController.cpp index 168604ee45..936f1c391f 100644 --- a/libraries/controllers/src/controllers/StandardController.cpp +++ b/libraries/controllers/src/controllers/StandardController.cpp @@ -399,7 +399,7 @@ Input::NamedVector StandardController::getAvailableInputs() const { makePair(PUFF, "Puff"), makePair(CHEEKSQUINT_L, "CheekSquint_L"), makePair(CHEEKSQUINT_R, "CheekSquint_R"), - makePair(LIPSTOGETHER, "LipsTogether"), + makePair(MOUTHCLOSE, "MouthClose"), makePair(MOUTHUPPERUP_L, "MouthUpperUp_L"), makePair(MOUTHUPPERUP_R, "MouthUpperUp_R"), makePair(MOUTHLOWERDOWN_L, "MouthLowerDown_L"), diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index f99072af7c..965c095187 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -134,7 +134,7 @@ namespace controller { PUFF, CHEEKSQUINT_L, CHEEKSQUINT_R, - LIPSTOGETHER, + MOUTHCLOSE, MOUTHUPPERUP_L, MOUTHUPPERUP_R, MOUTHLOWERDOWN_L, diff --git a/libraries/shared/src/BlendshapeConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp index 7564c31944..91b68ed8a9 100644 --- a/libraries/shared/src/BlendshapeConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -54,7 +54,7 @@ const char* FACESHIFT_BLENDSHAPES[] = { "Puff", "CheekSquint_L", "CheekSquint_R", - "LipsTogether", + "MouthClose", "MouthUpperUp_L", "MouthUpperUp_R", "MouthLowerDown_L", diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 0f934b2056..8db29856c3 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -58,7 +58,7 @@ enum class Blendshapes : int { Puff, CheekSquint_L, CheekSquint_R, - LipsTogether, + MouthClose, MouthUpperUp_L, MouthUpperUp_R, MouthLowerDown_L, @@ -94,7 +94,7 @@ enum class LegacyBlendshpaes : int { }; // NEW in ARKit -// * LipsTogether +// * MouthClose // * MouthUpperUp_L // * MouthUpperUp_R // * MouthLowerDown_L From 61b9f08fec914aa6a8233a398a545dfaa87e1da0 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 22 Oct 2019 22:19:03 -0700 Subject: [PATCH 07/19] Blendshapes for OtherAvatars should update even if no joints have changes. --- interface/src/avatar/OtherAvatar.cpp | 5 ++++- .../avatars-renderer/src/avatars-renderer/SkeletonModel.cpp | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/OtherAvatar.cpp b/interface/src/avatar/OtherAvatar.cpp index 5673c2443f..50f6369dbe 100755 --- a/interface/src/avatar/OtherAvatar.cpp +++ b/interface/src/avatar/OtherAvatar.cpp @@ -267,6 +267,7 @@ void OtherAvatar::simulate(float deltaTime, bool inView) { _skeletonModel->getRig().computeExternalPoses(rootTransform); _jointDataSimulationRate.increment(); + head->simulate(deltaTime); _skeletonModel->simulate(deltaTime, true); locationChanged(); // joints changed, so if there are any children, update them. @@ -277,9 +278,11 @@ void OtherAvatar::simulate(float deltaTime, bool inView) { headPosition = getWorldPosition(); } head->setPosition(headPosition); + } else { + head->simulate(deltaTime); + _skeletonModel->simulate(deltaTime, false); } head->setScale(getModelScale()); - head->simulate(deltaTime); relayJointDataToChildren(); } else { // a non-full update is still required so that the position, rotation, scale and bounds of the skeletonModel are updated. diff --git a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp index b52a68f066..fc16851942 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp @@ -165,8 +165,9 @@ void SkeletonModel::updateAttitude(const glm::quat& orientation) { // but just before head has been simulated. void SkeletonModel::simulate(float deltaTime, bool fullUpdate) { updateAttitude(_owningAvatar->getWorldOrientation()); + setBlendshapeCoefficients(_owningAvatar->getHead()->getSummedBlendshapeCoefficients()); + if (fullUpdate) { - setBlendshapeCoefficients(_owningAvatar->getHead()->getSummedBlendshapeCoefficients()); Parent::simulate(deltaTime, fullUpdate); From f28d317914448c90f0c2a1697de8832b5dc29975 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 23 Oct 2019 16:43:10 -0700 Subject: [PATCH 08/19] Clean up reamining issues --- interface/src/avatar/MySkeletonModel.cpp | 14 +++++--------- .../src/avatars-renderer/SkeletonModel.cpp | 11 ----------- libraries/avatars/src/AvatarData.cpp | 3 ++- libraries/avatars/src/HeadData.cpp | 9 +-------- 4 files changed, 8 insertions(+), 29 deletions(-) diff --git a/interface/src/avatar/MySkeletonModel.cpp b/interface/src/avatar/MySkeletonModel.cpp index 0d7bd3f3b3..6fe199aaba 100755 --- a/interface/src/avatar/MySkeletonModel.cpp +++ b/interface/src/avatar/MySkeletonModel.cpp @@ -112,23 +112,19 @@ static AnimPose computeHipsInSensorFrame(MyAvatar* myAvatar, bool isFlying) { void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { const HFMModel& hfmModel = getHFMModel(); + MyAvatar* myAvatar = static_cast(_owningAvatar); + assert(myAvatar); + Head* head = _owningAvatar->getHead(); - // AJT: blendshapes TODO: RE-enable this and avoid duplication with - // SkeletonModel::updateRig() - /* - bool eyePosesValid = !head->getHasProceduralEyeMovement(); + bool eyePosesValid = (myAvatar->getControllerPoseInSensorFrame(controller::Action::LEFT_EYE).isValid() || + myAvatar->getControllerPoseInSensorFrame(controller::Action::RIGHT_EYE).isValid()); glm::vec3 lookAt; if (eyePosesValid) { lookAt = head->getLookAtPosition(); // don't apply no-crosseyes code when eyes are being tracked } else { lookAt = avoidCrossedEyes(head->getLookAtPosition()); } - */ - glm::vec3 lookAt = avoidCrossedEyes(head->getLookAtPosition()); - - MyAvatar* myAvatar = static_cast(_owningAvatar); - assert(myAvatar); Rig::ControllerParameters params; diff --git a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp index fc16851942..ccc87c28f3 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp @@ -110,17 +110,6 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { assert(!_owningAvatar->isMyAvatar()); Head* head = _owningAvatar->getHead(); - - // AJT: blendshapes TODO: RE-enable this. but move into rig? - /* - bool eyePosesValid = !head->getHasProceduralEyeMovement(); - glm::vec3 lookAt; - if (eyePosesValid) { - lookAt = head->getLookAtPosition(); // don't apply no-crosseyes code etc when eyes are being tracked - } else { - lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); - } - */ glm::vec3 lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); // no need to call Model::updateRig() because otherAvatars get their joint state diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 87d6c57d59..818902968f 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2637,6 +2637,7 @@ enum class JsonAvatarFrameVersion : int { JointRotationsInAbsoluteFrame, JointDefaultPoseBits, JointUnscaledTranslations, + ARKitBlendshapes }; QJsonValue toJsonValue(const JointData& joint) { @@ -2681,7 +2682,7 @@ void AvatarData::avatarEntityDataToJson(QJsonObject& root) const { QJsonObject AvatarData::toJson() const { QJsonObject root; - root[JSON_AVATAR_VERSION] = (int)JsonAvatarFrameVersion::JointUnscaledTranslations; + root[JSON_AVATAR_VERSION] = (int)JsonAvatarFrameVersion::ARKitBlendshapes; if (!getSkeletonModelURL().isEmpty()) { root[JSON_AVATAR_BODY_MODEL] = getSkeletonModelURL().toString(); diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 93fb0190bb..5e8d5c457f 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -186,14 +186,7 @@ QJsonObject HeadData::toJson() const { void HeadData::fromJson(const QJsonObject& json) { if (json.contains(JSON_AVATAR_HEAD_BLENDSHAPE_COEFFICIENTS)) { auto jsonValue = json[JSON_AVATAR_HEAD_BLENDSHAPE_COEFFICIENTS]; - if (jsonValue.isArray()) { - QVector blendshapeCoefficients; - QJsonArray blendshapeCoefficientsJson = jsonValue.toArray(); - for (const auto& blendshapeCoefficient : blendshapeCoefficientsJson) { - blendshapeCoefficients.push_back((float)blendshapeCoefficient.toDouble()); - } - setBlendshapeCoefficients(blendshapeCoefficients); - } else if (jsonValue.isObject()) { + if (jsonValue.isObject()) { QJsonObject blendshapeCoefficientsJson = jsonValue.toObject(); for (const QString& name : blendshapeCoefficientsJson.keys()) { float value = (float)blendshapeCoefficientsJson[name].toDouble(); From dc7b611e8e8aded9b618868d1dcfef444dbe25eb Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Thu, 24 Oct 2019 09:53:11 -0700 Subject: [PATCH 09/19] documentation and warning fixes --- interface/src/avatar/MyAvatar.cpp | 2 -- interface/src/avatar/MyAvatar.h | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index f1829db8f6..52083af8f2 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -6535,7 +6535,6 @@ void MyAvatar::updateEyesLookAtPosition(float deltaTime) { updateLookAtTargetAvatar(); - bool isLookingAtSomeone = false; glm::vec3 lookAtSpot; const MyHead* myHead = getMyHead(); @@ -6596,7 +6595,6 @@ void MyAvatar::updateEyesLookAtPosition(float deltaTime) { avatar && avatar->getLookAtSnappingEnabled() && getLookAtSnappingEnabled(); if (haveLookAtCandidate && mutualLookAtSnappingEnabled) { // If I am looking at someone else, look directly at one of their eyes - isLookingAtSomeone = true; auto lookingAtHead = avatar->getHead(); const float MAXIMUM_FACE_ANGLE = 65.0f * RADIANS_PER_DEGREE; diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 953b7ef1dc..9197b005be 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -307,7 +307,7 @@ class MyAvatar : public Avatar { * @borrows Avatar.clearAvatarEntity as clearAvatarEntity * @borrows Avatar.hasScriptedBlendshapes as hasScriptedBlendshapes * @borrows Avatar.hasProceduralBlinkFaceMovement as hasProceduralBlinkFaceMovement - * @borrows Avatar.hasEyeFaceMovement as hasEyeFaceMovement + * @borrows Avatar.hasProceduralEyeFaceMovement as hasProceduralEyeFaceMovement * @borrows Avatar.hasAudioEnabledFaceMovement as hasAudioEnabledFaceMovement * @borrows Avatar.setSkeletonModelURL as setSkeletonModelURL * @borrows Avatar.getAttachmentData as getAttachmentData From 9325ea588a1b0fda23bea233eaa64ad924c2c5d1 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Sat, 26 Oct 2019 09:20:51 -0700 Subject: [PATCH 10/19] Re-added removed jsdoc comments for removed AvatarInput jsapi. With a comment that these properties/functions have been removed. --- interface/src/ui/AvatarInputs.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/interface/src/ui/AvatarInputs.h b/interface/src/ui/AvatarInputs.h index 40ee0ea7d4..3b0e57d037 100644 --- a/interface/src/ui/AvatarInputs.h +++ b/interface/src/ui/AvatarInputs.h @@ -32,6 +32,14 @@ class AvatarInputs : public QObject { * @hifi-client-entity * @hifi-avatar * + * @property {boolean} cameraEnabled - true if webcam face tracking is enabled, false if it is + * disabled. + * Read-only. + *

Deprecated: This property is deprecated and has been removed.

+ * @property {boolean} cameraMuted - true if webcam face tracking is muted (temporarily disabled), + * false it if isn't. + * Read-only. + *

Deprecated: This property is deprecated and has been removed.

* @property {boolean} ignoreRadiusEnabled - true if the privacy shield is enabled, false if it * is disabled. * Read-only. @@ -87,6 +95,20 @@ public slots: signals: + /**jsdoc + * Triggered when webcam face tracking is enabled or disabled. + * @deprecated This signal is deprecated and has been removed. + * @function AvatarInputs.cameraEnabledChanged + * @returns {Signal} + */ + + /**jsdoc + * Triggered when webcam face tracking is muted (temporarily disabled) or unmuted. + * @deprecated This signal is deprecated and has been removed. + * @function AvatarInputs.cameraMutedChanged + * @returns {Signal} + */ + /**jsdoc * Triggered when the display mode changes between desktop and HMD. * @function AvatarInputs.isHMDChanged @@ -157,6 +179,12 @@ protected: */ Q_INVOKABLE void resetSensors(); + /**jsdoc + * Toggles the muting (temporary disablement) of webcam face tracking on/off. + *

Deprecated: This function is deprecated and has been removed.

+ * @function AvatarInputs.toggleCameraMute + */ + private: void onAvatarEnteredIgnoreRadius(); void onAvatarLeftIgnoreRadius(); From 884cbd7de93370a0d9ed0ec053b87bc1c8475cf9 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 09:26:48 -0700 Subject: [PATCH 11/19] working on moving display plugin lookup to audiodevices --- interface/src/scripting/AudioDevices.cpp | 29 +++++++++- libraries/audio-client/src/AudioClient.cpp | 57 ++++++++----------- libraries/audio-client/src/AudioClient.h | 5 +- .../plugins/src/plugins/PluginManager.cpp | 8 +-- 4 files changed, 58 insertions(+), 41 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index 76c871fa5c..8737cd1296 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include "Application.h" @@ -68,6 +69,21 @@ static QString getTargetDevice(bool hmd, QAudio::Mode mode) { return deviceName; } + +static QString getHmdAudioDeviceName(QAudio::Mode mode) { + foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getAllDisplayPlugins()) { + if (displayPlugin && displayPlugin->isHmd()) { + if (mode == QAudio::AudioInput) { + return displayPlugin->getPreferredAudioInDevice(); + } else { + return displayPlugin->getPreferredAudioOutDevice(); + } + break; + } + } + return QString(); +} + Qt::ItemFlags AudioDeviceList::_flags { Qt::ItemIsSelectable | Qt::ItemIsEnabled }; AudioDeviceList::AudioDeviceList(QAudio::Mode mode) : _mode(mode) { @@ -263,6 +279,18 @@ void AudioDeviceList::onDevicesChanged(const QList& devices bool hmdIsSelected = false; bool desktopIsSelected = false; + //getting hmd mode + if (devices.size() > 0) { + auto mode = devices.first().getMode(); + QString name = getHmdAudioDeviceName(mode); + if (!name.isEmpty()) { + auto client = DependencyManager::get().data(); + QMetaObject::invokeMethod(client, "setHmdAudioName", + Q_ARG(QAudio::Mode, mode), + Q_ARG(const QString&, name)); + } + } + if (!_backupSelectedDesktopDeviceName.isEmpty() && !_backupSelectedHMDDeviceName.isEmpty()) { foreach(const HifiAudioDeviceInfo& deviceInfo, devices) { for (bool isHMD : {false, true}) { @@ -275,7 +303,6 @@ void AudioDeviceList::onDevicesChanged(const QList& devices _selectedDesktopDevice = deviceInfo; backupSelectedDeviceName.clear(); } - } } } diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index fe4fab8415..e966274a9b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -48,7 +48,6 @@ #include #include #include -#include #include "AudioClientLogging.h" #include "AudioLogging.h" @@ -83,38 +82,31 @@ Mutex _recordMutex; QString defaultAudioDeviceName(QAudio::Mode mode); -static QString getHmdAudioDeviceName(QAudio::Mode mode) { - foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getAllDisplayPlugins()) { - if (displayPlugin && displayPlugin->isHmd()) { - if (mode == QAudio::AudioInput) { - return displayPlugin->getPreferredAudioInDevice(); - } else { - return displayPlugin->getPreferredAudioOutDevice(); - } - break; - } +void AudioClient::setHmdAudioName(QAudio::Mode mode, const QString& name) { + if (mode == QAudio::AudioInput) { + _hmdInputName = name; + } else { + _hmdOutputName = name; } - return QString(); } // thread-safe -QList getAvailableDevices(QAudio::Mode mode) { +QList getAvailableDevices(QAudio::Mode mode, const QString& hmdName) { //get hmd device name prior to locking device mutex. in case of shutdown, this thread will be locked and audio client //cannot properly shut down. - QString hmdDeviceName = getHmdAudioDeviceName(mode); QString defDeviceName = defaultAudioDeviceName(mode); // NOTE: availableDevices() clobbers the Qt internal device list Lock lock(_deviceMutex); auto devices = QAudioDeviceInfo::availableDevices(mode); - + HifiAudioDeviceInfo defaultDesktopDevice; QList newDevices; for (auto& device : devices) { newDevices.push_back(HifiAudioDeviceInfo(device, false, mode)); if (device.deviceName() == defDeviceName.trimmed()) { defaultDesktopDevice = HifiAudioDeviceInfo(device, true, mode, HifiAudioDeviceInfo::desktop); - } + } } if (defaultDesktopDevice.getDevice().isNull()) { @@ -123,11 +115,10 @@ QList getAvailableDevices(QAudio::Mode mode) { defaultDesktopDevice = HifiAudioDeviceInfo(devices.first(), true, mode, HifiAudioDeviceInfo::desktop); } newDevices.push_front(defaultDesktopDevice); - - if (!hmdDeviceName.isNull() && !hmdDeviceName.isEmpty()) { + if (!hmdName.isNull()) { HifiAudioDeviceInfo hmdDevice; foreach(auto device, newDevices) { - if (device.getDevice().deviceName() == hmdDeviceName) { + if (device.getDevice().deviceName() == hmdName) { hmdDevice = HifiAudioDeviceInfo(device.getDevice(), true, mode, HifiAudioDeviceInfo::hmd); break; } @@ -149,9 +140,9 @@ void AudioClient::checkDevices() { return; } - auto inputDevices = getAvailableDevices(QAudio::AudioInput); - auto outputDevices = getAvailableDevices(QAudio::AudioOutput); - + auto inputDevices = getAvailableDevices(QAudio::AudioInput, _hmdInputName); + auto outputDevices = getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); + checkDefaultChanges(inputDevices); checkDefaultChanges(outputDevices); @@ -335,8 +326,8 @@ AudioClient::AudioClient() { connect(&_receivedAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioClient::handleMismatchAudioFormat); // initialize wasapi; if getAvailableDevices is called from the CheckDevicesThread before this, it will crash - getAvailableDevices(QAudio::AudioInput); - getAvailableDevices(QAudio::AudioOutput); + getAvailableDevices(QAudio::AudioInput, _hmdInputName); + getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); // start a thread to detect any device changes _checkDevicesTimer = new QTimer(this); @@ -436,9 +427,9 @@ void AudioClient::setAudioPaused(bool pause) { } } -HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, bool isHmd=false) { +HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, const QString& hmdName, bool isHmd=false) { HifiAudioDeviceInfo result; - foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode)) { + foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode,hmdName)) { if (audioDevice.deviceName().trimmed() == deviceName.trimmed()) { if ((!isHmd && audioDevice.getDeviceType() != HifiAudioDeviceInfo::hmd) || (isHmd && audioDevice.getDeviceType() != HifiAudioDeviceInfo::desktop)) { result = audioDevice; @@ -493,7 +484,7 @@ QString AudioClient::getWinDeviceName(wchar_t* guid) { #endif -HifiAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { +HifiAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode, const QString& hmdName) { QString deviceName = defaultAudioDeviceName(mode); #if defined (Q_OS_ANDROID) if (mode == QAudio::AudioInput) { @@ -509,7 +500,7 @@ HifiAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { } } #endif - return getNamedAudioDeviceForMode(mode, deviceName); + return getNamedAudioDeviceForMode(mode, deviceName, hmdName); } QString defaultAudioDeviceName(QAudio::Mode mode) { @@ -615,7 +606,8 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { } bool AudioClient::getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName) { - return (getNamedAudioDeviceForMode(mode, deviceName).deviceName() == deviceName); + QString hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; + return (getNamedAudioDeviceForMode(mode, deviceName, hmdName).deviceName() == deviceName); } @@ -778,11 +770,11 @@ void AudioClient::start() { _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(OUTPUT_CHANNEL_COUNT); - HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput); + HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput, _hmdInputName); qCDebug(audioclient) << "The default audio input device is" << inputDeviceInfo.deviceName(); bool inputFormatSupported = switchInputToAudioDevice(inputDeviceInfo); - HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); + HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput, _hmdOutputName); qCDebug(audioclient) << "The default audio output device is" << outputDeviceInfo.deviceName(); bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo); @@ -1021,7 +1013,8 @@ bool AudioClient::switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo } bool AudioClient::switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd) { - return switchAudioDevice(mode, getNamedAudioDeviceForMode(mode, deviceName, isHmd)); + QString hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; + return switchAudioDevice(mode, getNamedAudioDeviceForMode(mode, deviceName, hmdName, isHmd)); } void AudioClient::configureReverb() { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index ce63c43cff..9628de99e8 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -313,7 +313,7 @@ private: // background tasks void checkDevices(); void checkPeakValues(); - + void setHmdAudioName(QAudio::Mode mode, const QString& name); void outputFormatChanged(); void handleAudioInput(QByteArray& audioBuffer); void prepareLocalAudioInjectors(std::unique_ptr localAudioLock = nullptr); @@ -481,6 +481,9 @@ private: QList _inputDevices; QList _outputDevices; + QString _hmdInputName { QString() }; + QString _hmdOutputName{ QString() }; + AudioFileWav _audioFileWav; bool _hasReceivedFirstPacket { false }; diff --git a/libraries/plugins/src/plugins/PluginManager.cpp b/libraries/plugins/src/plugins/PluginManager.cpp index 660159d9bd..784de6bdea 100644 --- a/libraries/plugins/src/plugins/PluginManager.cpp +++ b/libraries/plugins/src/plugins/PluginManager.cpp @@ -224,13 +224,7 @@ const OculusPlatformPluginPointer PluginManager::getOculusPlatformPlugin() { } DisplayPluginList PluginManager::getAllDisplayPlugins() { - if (thread() != QThread::currentThread()) { - DisplayPluginList list; - QMetaObject::invokeMethod(this, "getAllDisplayPlugins", Qt::BlockingQueuedConnection, Q_RETURN_ARG(DisplayPluginList, list)); - return list; - } else { - return _displayPlugins; - } + return _displayPlugins; } const DisplayPluginList& PluginManager::getDisplayPlugins() { From 9370ce9b763bbacd94449e267d2b38fe4ef259d0 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 10:24:30 -0700 Subject: [PATCH 12/19] removed items --- libraries/audio-client/CMakeLists.txt | 1 - libraries/audio-client/src/AudioClient.h | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index e1d90334ff..6b88292dd4 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -6,7 +6,6 @@ setup_hifi_library(Network Multimedia ${PLATFORM_QT_COMPONENTS}) link_hifi_libraries(audio plugins) include_hifi_library_headers(shared) include_hifi_library_headers(networking) -include_hifi_library_headers(gpu) if (ANDROID) else () diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 9628de99e8..fadc599228 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -243,6 +243,8 @@ public slots: // calling with a null QAudioDevice will use the system default bool switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo& deviceInfo = HifiAudioDeviceInfo()); bool switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd); + void setHmdAudioName(QAudio::Mode mode, const QString& name); + // Qt opensles plugin is not able to detect when the headset is plugged in void setHeadsetPluggedIn(bool pluggedIn); @@ -313,7 +315,6 @@ private: // background tasks void checkDevices(); void checkPeakValues(); - void setHmdAudioName(QAudio::Mode mode, const QString& name); void outputFormatChanged(); void handleAudioInput(QByteArray& audioBuffer); void prepareLocalAudioInjectors(std::unique_ptr localAudioLock = nullptr); From 77f5b985a1a7c0c7e9431de2c397a416307d9308 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 11:19:59 -0700 Subject: [PATCH 13/19] added read/write lock to hmd name --- libraries/audio-client/src/AudioClient.cpp | 43 +++++++++++++++++----- libraries/audio-client/src/AudioClient.h | 2 +- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index e966274a9b..c81823d14b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -83,6 +83,7 @@ Mutex _recordMutex; QString defaultAudioDeviceName(QAudio::Mode mode); void AudioClient::setHmdAudioName(QAudio::Mode mode, const QString& name) { + QWriteLocker lock(&_hmdNameLock); if (mode == QAudio::AudioInput) { _hmdInputName = name; } else { @@ -140,8 +141,17 @@ void AudioClient::checkDevices() { return; } - auto inputDevices = getAvailableDevices(QAudio::AudioInput, _hmdInputName); - auto outputDevices = getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); + QString hmdInputName; + QString hmdOutputName; + + { + QReadLocker readLock(&_hmdNameLock); + hmdInputName = _hmdInputName; + hmdOutputName = _hmdOutputName; + } + + auto inputDevices = getAvailableDevices(QAudio::AudioInput, hmdInputName); + auto outputDevices = getAvailableDevices(QAudio::AudioOutput, hmdOutputName); checkDefaultChanges(inputDevices); checkDefaultChanges(outputDevices); @@ -325,10 +335,12 @@ AudioClient::AudioClient() { connect(&_receivedAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioClient::handleMismatchAudioFormat); - // initialize wasapi; if getAvailableDevices is called from the CheckDevicesThread before this, it will crash - getAvailableDevices(QAudio::AudioInput, _hmdInputName); - getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); - + { + QReadLocker readLock(&_hmdNameLock); + // initialize wasapi; if getAvailableDevices is called from the CheckDevicesThread before this, it will crash + getAvailableDevices(QAudio::AudioInput, _hmdInputName); + getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); + } // start a thread to detect any device changes _checkDevicesTimer = new QTimer(this); const unsigned long DEVICE_CHECK_INTERVAL_MSECS = 2 * 1000; @@ -606,6 +618,7 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { } bool AudioClient::getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName) { + QReadLocker readLock(&_hmdNameLock); QString hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; return (getNamedAudioDeviceForMode(mode, deviceName, hmdName).deviceName() == deviceName); } @@ -769,12 +782,20 @@ void AudioClient::start() { _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(OUTPUT_CHANNEL_COUNT); + + QString inputName; + QString outputName; + { + QReadLocker readLock(&_hmdNameLock); + inputName = _hmdInputName; + outputName = _hmdOutputName; + } - HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput, _hmdInputName); + HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput, inputName); qCDebug(audioclient) << "The default audio input device is" << inputDeviceInfo.deviceName(); bool inputFormatSupported = switchInputToAudioDevice(inputDeviceInfo); - HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput, _hmdOutputName); + HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput, outputName); qCDebug(audioclient) << "The default audio output device is" << outputDeviceInfo.deviceName(); bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo); @@ -1013,7 +1034,11 @@ bool AudioClient::switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo } bool AudioClient::switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd) { - QString hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; + QString hmdName; + { + QReadLocker readLock(&_hmdNameLock); + hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; + } return switchAudioDevice(mode, getNamedAudioDeviceForMode(mode, deviceName, hmdName, isHmd)); } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index fadc599228..4e6abd3509 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -508,7 +508,7 @@ private: #endif AudioSolo _solo; - + QReadWriteLock _hmdNameLock; Mutex _checkDevicesMutex; QTimer* _checkDevicesTimer { nullptr }; Mutex _checkPeakValuesMutex; From 7d5dcbfc77e3cf047504d48f428994689745e9ce Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 11:58:04 -0700 Subject: [PATCH 14/19] space cleanup --- interface/src/scripting/AudioDevices.cpp | 1 - libraries/audio-client/src/AudioClient.cpp | 2 +- libraries/audio-client/src/AudioClient.h | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index 8737cd1296..8a4e0925e3 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -69,7 +69,6 @@ static QString getTargetDevice(bool hmd, QAudio::Mode mode) { return deviceName; } - static QString getHmdAudioDeviceName(QAudio::Mode mode) { foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getAllDisplayPlugins()) { if (displayPlugin && displayPlugin->isHmd()) { diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index c81823d14b..4b7d642a71 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -116,6 +116,7 @@ QList getAvailableDevices(QAudio::Mode mode, const QString& defaultDesktopDevice = HifiAudioDeviceInfo(devices.first(), true, mode, HifiAudioDeviceInfo::desktop); } newDevices.push_front(defaultDesktopDevice); + if (!hmdName.isNull()) { HifiAudioDeviceInfo hmdDevice; foreach(auto device, newDevices) { @@ -143,7 +144,6 @@ void AudioClient::checkDevices() { QString hmdInputName; QString hmdOutputName; - { QReadLocker readLock(&_hmdNameLock); hmdInputName = _hmdInputName; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 4e6abd3509..dafa81081c 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -244,7 +244,6 @@ public slots: bool switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo& deviceInfo = HifiAudioDeviceInfo()); bool switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd); void setHmdAudioName(QAudio::Mode mode, const QString& name); - // Qt opensles plugin is not able to detect when the headset is plugged in void setHeadsetPluggedIn(bool pluggedIn); @@ -315,6 +314,7 @@ private: // background tasks void checkDevices(); void checkPeakValues(); + void outputFormatChanged(); void handleAudioInput(QByteArray& audioBuffer); void prepareLocalAudioInjectors(std::unique_ptr localAudioLock = nullptr); @@ -508,6 +508,7 @@ private: #endif AudioSolo _solo; + QReadWriteLock _hmdNameLock; Mutex _checkDevicesMutex; QTimer* _checkDevicesTimer { nullptr }; From ddfbb55e19fedae98d1aeba7d4d9a83299f8b025 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 15:36:43 -0700 Subject: [PATCH 15/19] fixed old ui filtering of audio devices --- interface/resources/qml/hifi/audio/Audio.qml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/interface/resources/qml/hifi/audio/Audio.qml b/interface/resources/qml/hifi/audio/Audio.qml index fccba12a8a..eef339b854 100644 --- a/interface/resources/qml/hifi/audio/Audio.qml +++ b/interface/resources/qml/hifi/audio/Audio.qml @@ -375,14 +375,14 @@ Rectangle { x: margins.paddings interactive: false; height: contentHeight; - spacing: 4; + clip: true; model: AudioScriptingInterface.devices.input; delegate: Item { width: rightMostInputLevelPos - margins.paddings*2 - height: margins.sizeCheckBox > checkBoxInput.implicitHeight ? - margins.sizeCheckBox : checkBoxInput.implicitHeight - + height: ((type != "hmd" && bar.currentIndex === 0) || (type != "desktop" && bar.currentIndex === 1)) ? + (margins.sizeCheckBox > checkBoxInput.implicitHeight ? margins.sizeCheckBox + 4 : checkBoxInput.implicitHeight + 4) : 0 + visible: (type != "hmd" && bar.currentIndex === 0) || (type != "desktop" && bar.currentIndex === 1) AudioControls.CheckBox { id: checkBoxInput anchors.left: parent.left @@ -470,13 +470,13 @@ Rectangle { height: contentHeight; anchors.top: outputDeviceHeader.bottom; anchors.topMargin: 10; - spacing: 4; clip: true; model: AudioScriptingInterface.devices.output; delegate: Item { width: rightMostInputLevelPos - height: margins.sizeCheckBox > checkBoxOutput.implicitHeight ? - margins.sizeCheckBox : checkBoxOutput.implicitHeight + height: ((type != "hmd" && bar.currentIndex === 0) || (type != "desktop" && bar.currentIndex === 1)) ? + (margins.sizeCheckBox > checkBoxOutput.implicitHeight ? margins.sizeCheckBox + 4 : checkBoxOutput.implicitHeight + 4) : 0 + visible: (type != "hmd" && bar.currentIndex === 0) || (type != "desktop" && bar.currentIndex === 1) AudioControls.CheckBox { id: checkBoxOutput From 39e10926606ef1768c9679f92f4423d4918b1f1f Mon Sep 17 00:00:00 2001 From: amer cerkic Date: Wed, 30 Oct 2019 11:34:41 -0700 Subject: [PATCH 16/19] working on hmd fix --- interface/src/scripting/AudioDevices.cpp | 28 +++++++++++------------- interface/src/scripting/AudioDevices.h | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index 8a4e0925e3..306d822ed4 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -271,24 +271,22 @@ std::shared_ptr getSimilarDevice(const QString& deviceNa return devices[minDistanceIndex]; } -void AudioDeviceList::onDevicesChanged(const QList& devices) { + +void AudioDeviceList::onDevicesChanged(QAudio::Mode mode, const QList& devices) { beginResetModel(); QList> newDevices; bool hmdIsSelected = false; bool desktopIsSelected = false; - //getting hmd mode - if (devices.size() > 0) { - auto mode = devices.first().getMode(); - QString name = getHmdAudioDeviceName(mode); - if (!name.isEmpty()) { - auto client = DependencyManager::get().data(); - QMetaObject::invokeMethod(client, "setHmdAudioName", - Q_ARG(QAudio::Mode, mode), - Q_ARG(const QString&, name)); - } + QString name = getHmdAudioDeviceName(mode); + if (!name.isEmpty()) { + auto client = DependencyManager::get().data(); + QMetaObject::invokeMethod(client, "setHmdAudioName", + Q_ARG(QAudio::Mode, mode), + Q_ARG(const QString&, name)); } + if (!_backupSelectedDesktopDeviceName.isEmpty() && !_backupSelectedHMDDeviceName.isEmpty()) { foreach(const HifiAudioDeviceInfo& deviceInfo, devices) { @@ -452,8 +450,8 @@ AudioDevices::AudioDevices(bool& contextIsHMD) : _contextIsHMD(contextIsHMD) { const QList& devicesOutput = client->getAudioDevices(QAudio::AudioOutput); //setup devices - _inputs.onDevicesChanged(devicesInput); - _outputs.onDevicesChanged(devicesOutput); + _inputs.onDevicesChanged(QAudio::AudioInput, devicesInput); + _outputs.onDevicesChanged(QAudio::AudioOutput,devicesOutput); } AudioDevices::~AudioDevices() {} @@ -552,14 +550,14 @@ void AudioDevices::onDevicesChanged(QAudio::Mode mode, const QList& devices); + void onDevicesChanged(QAudio::Mode mode, const QList& devices); protected: friend class AudioDevices; From 4b4b722875cd64aaa786d637aa5cb592586b0600 Mon Sep 17 00:00:00 2001 From: amer cerkic Date: Wed, 30 Oct 2019 16:27:19 -0700 Subject: [PATCH 17/19] fixed issues related to context switching and selection with hmd --- interface/src/scripting/AudioDevices.cpp | 36 ++++++++++++---------- libraries/audio-client/src/AudioClient.cpp | 18 +---------- 2 files changed, 21 insertions(+), 33 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index 306d822ed4..d66cb84dfe 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -69,18 +69,25 @@ static QString getTargetDevice(bool hmd, QAudio::Mode mode) { return deviceName; } -static QString getHmdAudioDeviceName(QAudio::Mode mode) { +static void checkHmdDefaultsChange(QAudio::Mode mode) { + QString name; foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getAllDisplayPlugins()) { if (displayPlugin && displayPlugin->isHmd()) { if (mode == QAudio::AudioInput) { - return displayPlugin->getPreferredAudioInDevice(); + name = displayPlugin->getPreferredAudioInDevice(); } else { - return displayPlugin->getPreferredAudioOutDevice(); + name = displayPlugin->getPreferredAudioOutDevice(); } break; } } - return QString(); + + if (!name.isEmpty()) { + auto client = DependencyManager::get().data(); + QMetaObject::invokeMethod(client, "setHmdAudioName", + Q_ARG(QAudio::Mode, mode), + Q_ARG(const QString&, name)); + } } Qt::ItemFlags AudioDeviceList::_flags { Qt::ItemIsSelectable | Qt::ItemIsEnabled }; @@ -278,16 +285,8 @@ void AudioDeviceList::onDevicesChanged(QAudio::Mode mode, const QList> newDevices; bool hmdIsSelected = false; bool desktopIsSelected = false; - - QString name = getHmdAudioDeviceName(mode); - if (!name.isEmpty()) { - auto client = DependencyManager::get().data(); - QMetaObject::invokeMethod(client, "setHmdAudioName", - Q_ARG(QAudio::Mode, mode), - Q_ARG(const QString&, name)); - } - + checkHmdDefaultsChange(mode); if (!_backupSelectedDesktopDeviceName.isEmpty() && !_backupSelectedHMDDeviceName.isEmpty()) { foreach(const HifiAudioDeviceInfo& deviceInfo, devices) { for (bool isHMD : {false, true}) { @@ -441,6 +440,9 @@ AudioDevices::AudioDevices(bool& contextIsHMD) : _contextIsHMD(contextIsHMD) { connect(client, &AudioClient::deviceChanged, this, &AudioDevices::onDeviceChanged, Qt::QueuedConnection); connect(client, &AudioClient::devicesChanged, this, &AudioDevices::onDevicesChanged, Qt::QueuedConnection); connect(client, &AudioClient::peakValueListChanged, &_inputs, &AudioInputDeviceList::onPeakValueListChanged, Qt::QueuedConnection); + + checkHmdDefaultsChange(QAudio::AudioInput); + checkHmdDefaultsChange(QAudio::AudioOutput); _inputs.onDeviceChanged(client->getActiveAudioDevice(QAudio::AudioInput), contextIsHMD); _outputs.onDeviceChanged(client->getActiveAudioDevice(QAudio::AudioOutput), contextIsHMD); @@ -449,9 +451,11 @@ AudioDevices::AudioDevices(bool& contextIsHMD) : _contextIsHMD(contextIsHMD) { const QList& devicesInput = client->getAudioDevices(QAudio::AudioInput); const QList& devicesOutput = client->getAudioDevices(QAudio::AudioOutput); - //setup devices - _inputs.onDevicesChanged(QAudio::AudioInput, devicesInput); - _outputs.onDevicesChanged(QAudio::AudioOutput,devicesOutput); + if (devicesInput.size() > 0 && devicesOutput.size() > 0) { + //setup devices + _inputs.onDevicesChanged(QAudio::AudioInput, devicesInput); + _outputs.onDevicesChanged(QAudio::AudioOutput, devicesOutput); + } } AudioDevices::~AudioDevices() {} diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 4b7d642a71..d8741e4aa7 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -791,23 +791,7 @@ void AudioClient::start() { outputName = _hmdOutputName; } - HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput, inputName); - qCDebug(audioclient) << "The default audio input device is" << inputDeviceInfo.deviceName(); - bool inputFormatSupported = switchInputToAudioDevice(inputDeviceInfo); - - HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput, outputName); - qCDebug(audioclient) << "The default audio output device is" << outputDeviceInfo.deviceName(); - bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo); - - if (!inputFormatSupported) { - qCDebug(audioclient) << "Unable to set up audio input because of a problem with input format."; - qCDebug(audioclient) << "The closest format available is" << inputDeviceInfo.getDevice().nearestFormat(_desiredInputFormat); - } - - if (!outputFormatSupported) { - qCDebug(audioclient) << "Unable to set up audio output because of a problem with output format."; - qCDebug(audioclient) << "The closest format available is" << outputDeviceInfo.getDevice().nearestFormat(_desiredOutputFormat); - } + #if defined(Q_OS_ANDROID) connect(&_checkInputTimer, &QTimer::timeout, this, &AudioClient::checkInputTimeout); _checkInputTimer.start(CHECK_INPUT_READS_MSECS); From 52e0f54d0c76456b7296bf10a321ba7c65817df2 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Thu, 31 Oct 2019 07:21:16 -0700 Subject: [PATCH 18/19] addressing comments --- interface/src/scripting/AudioDevices.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index d66cb84dfe..688b4df8cd 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -554,14 +554,14 @@ void AudioDevices::onDevicesChanged(QAudio::Mode mode, const QList Date: Thu, 31 Oct 2019 11:29:46 -0700 Subject: [PATCH 19/19] Attempt to fix DEV-2615 and DEV-2618 with more menu scripting interface guards --- .../src/scripting/MenuScriptingInterface.cpp | 119 +++++++++++++++--- 1 file changed, 100 insertions(+), 19 deletions(-) diff --git a/interface/src/scripting/MenuScriptingInterface.cpp b/interface/src/scripting/MenuScriptingInterface.cpp index 1020c12733..9e7f6fdc2b 100644 --- a/interface/src/scripting/MenuScriptingInterface.cpp +++ b/interface/src/scripting/MenuScriptingInterface.cpp @@ -32,106 +32,187 @@ void MenuScriptingInterface::menuItemTriggered() { } void MenuScriptingInterface::addMenu(const QString& menu, const QString& grouping) { - QMetaObject::invokeMethod(Menu::getInstance(), "addMenu", Q_ARG(const QString&, menu), Q_ARG(const QString&, grouping)); + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "addMenu", Q_ARG(const QString&, menu), Q_ARG(const QString&, grouping)); } void MenuScriptingInterface::removeMenu(const QString& menu) { - QMetaObject::invokeMethod(Menu::getInstance(), "removeMenu", Q_ARG(const QString&, menu)); + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "removeMenu", Q_ARG(const QString&, menu)); } bool MenuScriptingInterface::menuExists(const QString& menu) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return false; + } + if (QThread::currentThread() == qApp->thread()) { - Menu* menuInstance = Menu::getInstance(); return menuInstance && menuInstance->menuExists(menu); } + bool result { false }; - BLOCKING_INVOKE_METHOD(Menu::getInstance(), "menuExists", + + BLOCKING_INVOKE_METHOD(menuInstance, "menuExists", Q_RETURN_ARG(bool, result), Q_ARG(const QString&, menu)); + return result; } void MenuScriptingInterface::addSeparator(const QString& menuName, const QString& separatorName) { - QMetaObject::invokeMethod(Menu::getInstance(), "addSeparator", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "addSeparator", Q_ARG(const QString&, menuName), Q_ARG(const QString&, separatorName)); } void MenuScriptingInterface::removeSeparator(const QString& menuName, const QString& separatorName) { - QMetaObject::invokeMethod(Menu::getInstance(), "removeSeparator", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "removeSeparator", Q_ARG(const QString&, menuName), Q_ARG(const QString&, separatorName)); } void MenuScriptingInterface::addMenuItem(const MenuItemProperties& properties) { - QMetaObject::invokeMethod(Menu::getInstance(), "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); } void MenuScriptingInterface::addMenuItem(const QString& menu, const QString& menuitem, const QString& shortcutKey) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + MenuItemProperties properties(menu, menuitem, shortcutKey); - QMetaObject::invokeMethod(Menu::getInstance(), "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); + QMetaObject::invokeMethod(menuInstance, "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); } void MenuScriptingInterface::addMenuItem(const QString& menu, const QString& menuitem) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + MenuItemProperties properties(menu, menuitem); - QMetaObject::invokeMethod(Menu::getInstance(), "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); + QMetaObject::invokeMethod(menuInstance, "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); } void MenuScriptingInterface::removeMenuItem(const QString& menu, const QString& menuitem) { - QMetaObject::invokeMethod(Menu::getInstance(), "removeMenuItem", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + QMetaObject::invokeMethod(menuInstance, "removeMenuItem", Q_ARG(const QString&, menu), Q_ARG(const QString&, menuitem)); }; bool MenuScriptingInterface::menuItemExists(const QString& menu, const QString& menuitem) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return false; + } + if (QThread::currentThread() == qApp->thread()) { - Menu* menuInstance = Menu::getInstance(); return menuInstance && menuInstance->menuItemExists(menu, menuitem); } + bool result { false }; - BLOCKING_INVOKE_METHOD(Menu::getInstance(), "menuItemExists", + + BLOCKING_INVOKE_METHOD(menuInstance, "menuItemExists", Q_RETURN_ARG(bool, result), Q_ARG(const QString&, menu), Q_ARG(const QString&, menuitem)); + return result; } bool MenuScriptingInterface::isOptionChecked(const QString& menuOption) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return false; + } + if (QThread::currentThread() == qApp->thread()) { - Menu* menuInstance = Menu::getInstance(); return menuInstance && menuInstance->isOptionChecked(menuOption); } + bool result { false }; - BLOCKING_INVOKE_METHOD(Menu::getInstance(), "isOptionChecked", + + BLOCKING_INVOKE_METHOD(menuInstance, "isOptionChecked", Q_RETURN_ARG(bool, result), Q_ARG(const QString&, menuOption)); return result; } void MenuScriptingInterface::setIsOptionChecked(const QString& menuOption, bool isChecked) { - QMetaObject::invokeMethod(Menu::getInstance(), "setIsOptionChecked", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "setIsOptionChecked", Q_ARG(const QString&, menuOption), Q_ARG(bool, isChecked)); } bool MenuScriptingInterface::isMenuEnabled(const QString& menuOption) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return false; + } + if (QThread::currentThread() == qApp->thread()) { - Menu* menuInstance = Menu::getInstance(); return menuInstance && menuInstance->isMenuEnabled(menuOption); } + bool result { false }; - BLOCKING_INVOKE_METHOD(Menu::getInstance(), "isMenuEnabled", + + BLOCKING_INVOKE_METHOD(menuInstance, "isMenuEnabled", Q_RETURN_ARG(bool, result), Q_ARG(const QString&, menuOption)); + return result; } void MenuScriptingInterface::setMenuEnabled(const QString& menuOption, bool isChecked) { - QMetaObject::invokeMethod(Menu::getInstance(), "setMenuEnabled", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "setMenuEnabled", Q_ARG(const QString&, menuOption), Q_ARG(bool, isChecked)); } void MenuScriptingInterface::triggerOption(const QString& menuOption) { - QMetaObject::invokeMethod(Menu::getInstance(), "triggerOption", Q_ARG(const QString&, menuOption)); + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "triggerOption", Q_ARG(const QString&, menuOption)); }