From e4bc3a7d9d59a3050685ef86c6f4bc123d253187 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 12 Oct 2019 07:58:13 +1300 Subject: [PATCH 01/33] Recording JSDoc --- .../src/RecordingScriptingInterface.h | 178 +++++++++++++++--- 1 file changed, 147 insertions(+), 31 deletions(-) diff --git a/libraries/script-engine/src/RecordingScriptingInterface.h b/libraries/script-engine/src/RecordingScriptingInterface.h index 604ec6bc2e..19db6dc50d 100644 --- a/libraries/script-engine/src/RecordingScriptingInterface.h +++ b/libraries/script-engine/src/RecordingScriptingInterface.h @@ -24,6 +24,9 @@ class QScriptEngine; class QScriptValue; /**jsdoc + * The Recording API makes and plays back recordings of voice and avatar movements. Playback may be done on a + * user's avatar or an assignment client agent (see the {@link Agent} API). + * * @namespace Recording * * @hifi-interface @@ -40,56 +43,79 @@ public: public slots: /**jsdoc - * @function Recording.loadRecording - * @param {string} url - * @param {Recording~loadRecordingCallback} [callback=null] + * Called when a {@link Recording.loadRecording} call is complete. + * @callback Recording~loadRecordingCallback + * @param {boolean} success - true if the recording has successfully been loaded, false if it + * hasn't. + * @param {string} url - The URL of the recording that was requested to be loaded. */ /**jsdoc - * Called when {@link Recording.loadRecording} is complete. - * @callback Recording~loadRecordingCallback - * @param {boolean} success - * @param {string} url + * Loads a recording so that it is ready for playing. + * @function Recording.loadRecording + * @param {string} url - The ATP, HTTP, or file system URL of the recording to load. + * @param {Recording~loadRecordingCallback} [callback=null] - The function to call upon completion. + * @example Load and play back a recording from the asset server. + * var assetPath = Window.browseAssets(); + * print("Asset path: " + assetPath); + * + * if (assetPath.slice(-4) === ".hfr") { + * Recording.loadRecording("atp:" + assetPath, function (success, url) { + * if (!success) { + * print("Error loading recording."); + * return; + * } + * Recording.startPlaying(); + * }); + * } */ void loadRecording(const QString& url, QScriptValue callback = QScriptValue()); /**jsdoc + * Starts playing the recording currently loaded or paused. * @function Recording.startPlaying */ void startPlaying(); /**jsdoc + * Pauses playback of the recording currently playing. Use {@link Recording.startPlaying|startPlaying} to resume playback + * or {@link Recording.stopPlaying|stopPlaying} to stop playback. * @function Recording.pausePlayer */ void pausePlayer(); /**jsdoc + * Stops playing the recording currently playing or paused. * @function Recording.stopPlaying */ void stopPlaying(); /**jsdoc + * Gets whether a recording is currently playing. * @function Recording.isPlaying - * @returns {boolean} + * @returns {boolean} true if a recording is being played, false if one isn't. */ bool isPlaying() const; /**jsdoc + * Gets whether recording playback is currently paused. * @function Recording.isPaused - * @returns {boolean} + * @returns {boolean} true if recording playback is currently paused, false if it isn't. */ bool isPaused() const; /**jsdoc + * Gets the current playback time in the loaded recording, in seconds. * @function Recording.playerElapsed - * @returns {number} + * @returns {number} The current playback time in the loaded recording, in seconds. */ float playerElapsed() const; /**jsdoc + * Gets the length of the loaded recording, in seconds. * @function Recording.playerLength - * @returns {number} + * @returns {number} The length of the recording currently loaded, in seconds */ float playerLength() const; @@ -102,132 +128,222 @@ public slots: void setPlayerVolume(float volume); /**jsdoc + *

Not implemented: This method is not implemented yet.

* @function Recording.setPlayerAudioOffset - * @param {number} audioOffset + * @param {number} audioOffset - Audio offset. */ void setPlayerAudioOffset(float audioOffset); /**jsdoc + * Sets the current playback time in the loaded recording. * @function Recording.setPlayerTime - * @param {number} time + * @param {number} time - The current playback time, in seconds. */ void setPlayerTime(float time); /**jsdoc + * Sets whether playback should repeat in a loop. * @function Recording.setPlayerLoop - * @param {boolean} loop + * @param {boolean} loop - true if playback should repeat, false if it shouldn't. */ void setPlayerLoop(bool loop); /**jsdoc + * Sets whether recording playback will use the display name that the recording was made with. * @function Recording.setPlayerUseDisplayName - * @param {boolean} useDisplayName + * @param {boolean} useDisplayName - true to have recording playback use the display name that the recording + * was made with, false to have recording playback keep the current display name. */ void setPlayerUseDisplayName(bool useDisplayName); /**jsdoc + *

Not used.

* @function Recording.setPlayerUseAttachments - * @param {boolean} useAttachments + * @param {boolean} useAttachments - Use attachments. + * @deprecated This method is deprecated and will be removed. */ void setPlayerUseAttachments(bool useAttachments); /**jsdoc + *

Not used.

* @function Recording.setPlayerUseHeadModel - * @param {boolean} useHeadModel - * @todo Note: This function currently has no effect. + * @param {boolean} useHeadModel - Use head model. + * @deprecated This method is deprecated and will be removed. */ void setPlayerUseHeadModel(bool useHeadModel); /**jsdoc + * Sets whether recording playback will use the avatar model that the recording was made with. * @function Recording.setPlayerUseSkeletonModel - * @param {boolean} useSkeletonModel - * @todo Note: This function currently doesn't work. + * @param {boolean} useSkeletonModel - true to have recording playback use the avatar model that the recording + * was made with, false to have playback use the current avatar model. */ void setPlayerUseSkeletonModel(bool useSkeletonModel); /**jsdoc + * Sets whether recordings are played at the current avatar location or the recorded location. * @function Recording.setPlayFromCurrentLocation - * @param {boolean} playFromCurrentLocation + * @param {boolean} playFromCurrentLocation - true to play recordings at the current avatar location, + * false to play recordings at the recorded location. */ void setPlayFromCurrentLocation(bool playFromCurrentLocation); /**jsdoc + * Gets whether recording playback will use the display name that the recording was made with. * @function Recording.getPlayerUseDisplayName - * @returns {boolean} + * @returns {boolean} true if recording playback will use the display name that the recording was made with, + * false if playback will keep the current display name. */ bool getPlayerUseDisplayName() { return _useDisplayName; } /**jsdoc + *

Not used.

* @function Recording.getPlayerUseAttachments - * @returns {boolean} + * @returns {boolean} Use attachments. + * @deprecated This method is deprecated and will be removed. */ bool getPlayerUseAttachments() { return _useAttachments; } /**jsdoc + *

Not used.

* @function Recording.getPlayerUseHeadModel - * @returns {boolean} + * @returns {boolean} Use head model. + * @deprecated This method is deprecated and will be removed. */ bool getPlayerUseHeadModel() { return _useHeadModel; } /**jsdoc + * Gets whether recording playback will use the avatar model that the recording was made with. * @function Recording.getPlayerUseSkeletonModel - * @returns {boolean} + * @returns {boolean} true if recording playback will use the avatar model that the recording was made with, + * false if playback will use the current avatar model. */ bool getPlayerUseSkeletonModel() { return _useSkeletonModel; } /**jsdoc + * Gets whether recordings are played at the current avatar location or the recorded location. * @function Recording.getPlayFromCurrentLocation - * @returns {boolean} + * @returns {boolean} true if recordings are played at the current avatar location, false if + * played at the recorded location. */ bool getPlayFromCurrentLocation() { return _playFromCurrentLocation; } /**jsdoc + * Starts making a recording. * @function Recording.startRecording */ void startRecording(); /**jsdoc + * Stops making a recording. The recording may be saved using {@link Recording.saveRecording|saveRecording} or + * {@link Recording.saveRecordingToAsset|saveRecordingToAsset}, or immediately played back with + * {@link Recording.loadLastRecording|loadLastRecording}. * @function Recording.stopRecording */ void stopRecording(); /**jsdoc + * Gets whether a recording is currently being made. * @function Recording.isRecording - * @returns {boolean} + * @returns {boolean} true if a recording is currently being made, false if one isn't. */ bool isRecording() const; /**jsdoc + * Gets the duration of the recording currently being made or recently made, in seconds. * @function Recording.recorderElapsed - * @returns {number} + * @returns {number} The duration of the recording currently being made or recently made, in seconds. */ float recorderElapsed() const; /**jsdoc + * Gets the default directory that recordings are saved in. * @function Recording.getDefaultRecordingSaveDirectory - * @returns {string} + * @returns {string} The default recording save directory. + * @exampl Report the default save directory. + * print("Default save directory: " + Recording.getDefaultRecordingSaveDirectory()); */ QString getDefaultRecordingSaveDirectory(); /**jsdoc + * Saves the most recently made recording to a file. * @function Recording.saveRecording - * @param {string} filename + * @param {string} filename - The path and name of the file to save the recording to. + * @example Save a 5 second recording to a file. + * Recording.startRecording(); + * + * Script.setTimeout(function () { + * Recording.stopRecording(); + * var filename = (new Date()).toISOString(); // yyyy-mm-ddThh:mm:ss.sssZ + * filename = filename.slice(0, -5).replace(/:/g, "").replace("T", "-") + * + ".hfr"; // yyyymmmdd-hhmmss.hfr + * filename = Recording.getDefaultRecordingSaveDirectory() + filename; + * Recording.saveRecording(filename); + * print("Saved recording: " + filename); + * }, 5000); */ void saveRecording(const QString& filename); /**jsdoc + * Called when a {@link Recording.saveRecordingToAsset} call is complete. + * @callback Recording~saveRecordingToAssetCallback + * @param {string} url - The URL of the recording stored in the asset server if successful, "" if + * unsuccessful. The URL has atp: as the scheme and the SHA256 hash as the filename (with no extension). + */ + /**jsdoc + * Saves the most recently made recording to the domain's asset server. * @function Recording.saveRecordingToAsset - * @param {function} getClipAtpUrl + * @param {Recording~saveRecordingToAssetCallback} callback - The function to call upon completion. + * @returns {boolean} true if the recording is successfully being saved, false if not. + * @example Save a 5 second recording to the asset server. + * function onSavedRecordingToAsset(url) { + * if (url === "") { + * print("Couldn't save recording."); + * return; + * } + * + * print("Saved recording: " + url); // atp:SHA256 + * + * var filename = (new Date()).toISOString(); // yyyy-mm-ddThh:mm:ss.sssZ + * filename = filename.slice(0, -5).replace(/:/g, "").replace("T", "-") + * + ".hfr"; // yyyymmmdd-hhmmss.hfr + * var hash = url.slice(4); // Remove leading "atp:" from url. + * mappingPath = "/recordings/" + filename; + * Assets.setMapping(mappingPath, hash, function (error) { + * if (error) { + * print("Mapping error: " + error); + * } + * }); + * print("Mapped recording: " + mappingPath); // /recordings/filename + * } + * + * Recording.startRecording(); + * + * Script.setTimeout(function () { + * Recording.stopRecording(); + * var success = Recording.saveRecordingToAsset(onSavedRecordingToAsset); + * if (!success) { + * print("Couldn't save recording."); + * } + * }, 5000); */ bool saveRecordingToAsset(QScriptValue getClipAtpUrl); /**jsdoc + * Loads the most recently made recording and plays it back on your avatar. * @function Recording.loadLastRecording + * @example Make a 5 second recording and immediately play it back on your avatar. + * Recording.startRecording(); + * + * Script.setTimeout(function () { + * Recording.stopRecording(); + * Recording.loadLastRecording(); + * }, 5000); */ void loadLastRecording(); From 1715a86dbaf629fa1853312de3d844e76650e45f Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 12 Oct 2019 07:58:26 +1300 Subject: [PATCH 02/33] Fix typo in Assets JSDoc --- libraries/script-engine/src/AssetScriptingInterface.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/script-engine/src/AssetScriptingInterface.h b/libraries/script-engine/src/AssetScriptingInterface.h index 955adaa86c..3ba7632cd0 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.h +++ b/libraries/script-engine/src/AssetScriptingInterface.h @@ -30,8 +30,8 @@ * format: atp:/path/filename. The assets may optionally be baked, in which case a request for the original * unbaked version of the asset is automatically redirected to the baked version. The asset data may optionally be stored as * compressed.

- *

The client cache can be access directly, using "atp:" or "cache:" URLs. Interface, avatar, and - * assignment client scripts can write to the cache. All script types can read from the cache.

+ *

The client cache can be accessed directly, using "atp:" or "cache:" URLs. Interface, avatar, + * and assignment client scripts can write to the cache. All script types can read from the cache.

* * @namespace Assets * From 94f9e35704637832b5d493ecf8b522ccce8ba448 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 12 Oct 2019 11:41:53 +1300 Subject: [PATCH 03/33] Toolbars JSDoc --- .../ui/src/ui/ToolbarScriptingInterface.h | 110 +++++++++--------- 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/libraries/ui/src/ui/ToolbarScriptingInterface.h b/libraries/ui/src/ui/ToolbarScriptingInterface.h index 952d3cce95..a4746e4566 100644 --- a/libraries/ui/src/ui/ToolbarScriptingInterface.h +++ b/libraries/ui/src/ui/ToolbarScriptingInterface.h @@ -19,57 +19,16 @@ class QQuickItem; -/**jsdoc - * @class ToolbarButtonProxy - * - * @hifi-interface - * @hifi-client-entity - * @hifi-avatar - */ +// No JSDoc for ToolbarButtonProxy because oolbarProxy#addButton() doesn't work. class ToolbarButtonProxy : public QmlWrapper { Q_OBJECT public: ToolbarButtonProxy(QObject* qmlObject, QObject* parent = nullptr); - /**jsdoc - * @function ToolbarButtonProxy#editProperties - * @param {object} properties - */ Q_INVOKABLE void editProperties(const QVariantMap& properties); - - // QmlWrapper methods. - - /**jsdoc - * @function ToolbarButtonProxy#writeProperty - * @parm {string} propertyName - * @param {object} propertyValue - */ - - /**jsdoc - * @function ToolbarButtonProxy#writeProperties - * @param {object} properties - */ - - /**jsdoc - * @function ToolbarButtonProxy#readProperty - * @param {string} propertyName - * @returns {object} - */ - - /**jsdoc - * @function ToolbarButtonProxy#readProperties - * @param {string[]} propertyList - * @returns {object} - */ - signals: - - /**jsdoc - * @function ToolbarButtonProxy#clicked - * @returns {Signal} - */ void clicked(); protected: @@ -80,7 +39,12 @@ protected: Q_DECLARE_METATYPE(ToolbarButtonProxy*); /**jsdoc + * An instance of a toolbar. + * + *

Retrieve an existing toolbar or create a new toolbar using {@link Toolbars.getToolbar}.

+ * * @class ToolbarProxy + * @hideconstructor * * @hifi-interface * @hifi-client-entity @@ -92,15 +56,19 @@ public: ToolbarProxy(QObject* qmlObject, QObject* parent = nullptr); /**jsdoc + * Currently doesn't work. * @function ToolbarProxy#addButton - * @param {object} properties - * @returns {ToolbarButtonProxy} + * @param {object} properties - Button properties + * @returns {object} The button added. + * @deprecated This method is deprecated and will be removed. */ Q_INVOKABLE ToolbarButtonProxy* addButton(const QVariant& properties); /**jsdoc + * Currently doesn't work. * @function ToolbarProxy#removeButton - * @param {string} name + * @param {string} name - Button name. + * @deprecated This method is deprecated and will be removed. */ Q_INVOKABLE void removeButton(const QVariant& name); @@ -108,32 +76,46 @@ public: // QmlWrapper methods. /**jsdoc + * Sets the value of a toolbar property. A property is added to the toolbar if the named property doesn't already + * exist. * @function ToolbarProxy#writeProperty - * @parm {string} propertyName - * @param {object} propertyValue + * @parm {string} propertyName - The name of the property. Toolbar properties are those in the QML implementation of the + * toolbar. + * @param {object} propertyValue - The value of the property. */ /**jsdoc + * Sets the values of toolbar properties. A property is added to the toolbar if a named property doesn't already + * exist. * @function ToolbarProxy#writeProperties - * @param {object} properties + * @param {object} properties - The names and values of the properties to set. Toolbar properties are those in the QML + * implementation of the toolbar. */ /**jsdoc + * Gets the value of a toolbar property. * @function ToolbarProxy#readProperty - * @param {string} propertyName - * @returns {object} + * @param {string} propertyName - The property name. Toolbar properties are those in the QML implementation of the toolbar. + * @returns {object} The value of the property if the property name is valid, otherwise undefined. */ /**jsdoc + * Gets the values of toolbar properties. * @function ToolbarProxy#readProperties - * @param {string[]} propertyList - * @returns {object} + * @param {string[]} propertyList - The names of the properties to get the values of. Toolbar properties are those in the + * QML implementation of the toolbar. + * @returns {object} The names and values of the specified properties. If the toolbar doesn't have a particular property + * then the result doesn't include that property. */ }; Q_DECLARE_METATYPE(ToolbarProxy*); /**jsdoc + * The Toolbars API provides facilities to work with the system or other toolbar. + * + *

See also the {@link Tablet} API for use of the system tablet and toolbar in desktop and HMD modes.

+ * * @namespace Toolbars * * @hifi-interface @@ -145,13 +127,33 @@ class ToolbarScriptingInterface : public QObject, public Dependency { public: /**jsdoc + * Gets an instance of a toolbar. A new toolbar is created if one with the specified name doesn't already exist. * @function Toolbars.getToolbar - * @param {string} toolbarID - * @returns {ToolbarProxy} + * @param {string} name - A unique name that identifies the toolbar. + * @returns {ToolbarProxy} The toolbar instance. */ Q_INVOKABLE ToolbarProxy* getToolbar(const QString& toolbarId); signals: + /**jsdoc + * Triggered when the visibility of a toolbar changes. + * @function Toolbars.toolbarVisibleChanged + * @param {boolean} isVisible - true if the toolbar is visible, false if it is hidden. + * @param {string} toolbarName - The name of the toolbar. + * @returns {Signal} + * @example Briefly hide the system toolbar. + * Toolbars.toolbarVisibleChanged.connect(function(visible, name) { + * print("Toolbar " + name + " visible changed to " + visible); + * }); + * + * var toolbar = Toolbars.getToolbar("com.highfidelity.interface.toolbar.system"); + * if (toolbar) { + * toolbar.writeProperty("visible", false); + * Script.setTimeout(function () { + * toolbar.writeProperty("visible", true); + * }, 2000); + * } + */ void toolbarVisibleChanged(bool isVisible, QString toolbarName); }; From d4c5a20c4fe5cfffc41abe39f3211596efaeffe4 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 12 Oct 2019 11:42:49 +1300 Subject: [PATCH 04/33] Tablet JSDoc fixes --- libraries/ui/src/ui/TabletScriptingInterface.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libraries/ui/src/ui/TabletScriptingInterface.h b/libraries/ui/src/ui/TabletScriptingInterface.h index 88d4ebe267..04222b3ea1 100644 --- a/libraries/ui/src/ui/TabletScriptingInterface.h +++ b/libraries/ui/src/ui/TabletScriptingInterface.h @@ -39,9 +39,10 @@ class QmlWindowClass; class OffscreenQmlSurface; /**jsdoc - * The Tablet API provides the facilities to work with the system or other tablet. In toolbar mode (Developer > - * UI > Tablet Becomes Toolbar), the tablet's menu buttons are displayed in a toolbar and other tablet content is displayed - * in a dialog. + * The Tablet API provides the facilities to work with the system or other tablet. In toolbar mode (see Developer + * > UI options), the tablet's menu buttons are displayed in a toolbar and other tablet content is displayed in a dialog. + * + *

See also the {@link Toolbars} API for working with toolbars.

* * @namespace Tablet * @@ -98,7 +99,7 @@ public: void setToolbarScriptingInterface(ToolbarScriptingInterface* toolbarScriptingInterface) { _toolbarScriptingInterface = toolbarScriptingInterface; } /**jsdoc - * Gets an instance of a tablet. A new tablet is created if one with the specified ID doesn't already exist. + * Gets an instance of a tablet. A new tablet is created if one with the specified name doesn't already exist. * @function Tablet.getTablet * @param {string} name - A unique name that identifies the tablet. * @returns {TabletProxy} The tablet instance. @@ -210,11 +211,10 @@ private: Q_DECLARE_METATYPE(TabletButtonsProxyModel*); /**jsdoc - * An instance of a tablet. In toolbar mode (Developer > - * UI > Tablet Becomes Toolbar), the tablet's menu buttons are displayed in a toolbar and other tablet content is displayed - * in a dialog. + * An instance of a tablet. In toolbar mode (see Developer > UI options), the tablet's menu buttons are displayed in a + * toolbar and other tablet content is displayed in a dialog. * - *

Create a new tablet or retrieve an existing tablet using {@link Tablet.getTablet}.

+ *

Retrieve an existing tablet or create a new tablet using {@link Tablet.getTablet}.

* * @class TabletProxy * @@ -317,7 +317,7 @@ public: Q_INVOKABLE void returnToPreviousAppImpl(bool localSafeContext); /**jsdoc - *@function TabletProxy#loadQMLOnTopImpl + * @function TabletProxy#loadQMLOnTopImpl * @deprecated This function is deprecated and will be removed. */ // Internal function, do not call from scripts. From 8411e6b03351347e51f4af8176511f42e3823ff7 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 8 Oct 2019 10:42:19 -0700 Subject: [PATCH 05/33] Blendshape script API work * Moved hasScriptedBlendshapes, hasProceduralBlinkFaceMovement, hasProceduralEyeFaceMovement, hasAudioEnabledFaceMovement to AvatarData so they are accessable via agent scripts. * Marked setForceFaceTrackerConnected as depricated. * Updated jsdoc comments --- assignment-client/src/Agent.cpp | 1 - interface/src/avatar/MyAvatar.cpp | 25 ---------- interface/src/avatar/MyAvatar.h | 23 ++------- interface/src/avatar/MyHead.cpp | 4 +- libraries/avatars/src/AvatarData.cpp | 44 +++++++++++++---- libraries/avatars/src/AvatarData.h | 53 +++++++++++++++------ libraries/avatars/src/HeadData.cpp | 7 ++- libraries/avatars/src/HeadData.h | 6 +-- libraries/shared/src/FaceshiftConstants.cpp | 16 +++++-- 9 files changed, 98 insertions(+), 81 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index ee1e21c837..2cad2ca722 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -376,7 +376,6 @@ void Agent::executeScript() { // setup an Avatar for the script to use auto scriptedAvatar = DependencyManager::get(); scriptedAvatar->setID(getSessionUUID()); - scriptedAvatar->setForceFaceTrackerConnected(true); // call model URL setters with empty URLs so our avatar, if user, will have the default models scriptedAvatar->setSkeletonModelURL(QUrl()); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index a9ebd110d1..3c27ef52c2 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -3405,31 +3405,6 @@ bool MyAvatar::shouldRenderHead(const RenderArgs* renderArgs) const { return !defaultMode || (!firstPerson && !insideHead) || (overrideAnim && !insideHead); } -void MyAvatar::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { - if (hasScriptedBlendshapes == _hasScriptedBlendShapes) { - return; - } - if (!hasScriptedBlendshapes) { - // send a forced avatarData update to make sure the script can send neutal blendshapes on unload - // without having to wait for the update loop, make sure _hasScriptedBlendShapes is still true - // before sending the update, or else it won't send the neutal blendshapes to the receiving clients - sendAvatarDataPacket(true); - } - _hasScriptedBlendShapes = hasScriptedBlendshapes; -} - -void MyAvatar::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); -} - -void MyAvatar::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); -} - -void MyAvatar::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); -} - void MyAvatar::setRotationRecenterFilterLength(float length) { const float MINIMUM_ROTATION_RECENTER_FILTER_LENGTH = 0.01f; _rotationRecenterFilterLength = std::max(MINIMUM_ROTATION_RECENTER_FILTER_LENGTH, length); diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 0f139ddbff..acbd0f27a5 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -184,12 +184,6 @@ class MyAvatar : public Avatar { * property value is audioListenerModeCustom. * @property {Quat} customListenOrientation=Quat.IDENTITY - The listening orientation used when the * audioListenerMode property value is audioListenerModeCustom. - * @property {boolean} hasScriptedBlendshapes=false - true to transmit blendshapes over the network. - *

Note: Currently doesn't work. Use {@link MyAvatar.setForceFaceTrackerConnected} instead.

- * @property {boolean} hasProceduralBlinkFaceMovement=true - true if procedural blinking is turned on. - * @property {boolean} hasProceduralEyeFaceMovement=true - true if procedural eye movement is turned on. - * @property {boolean} hasAudioEnabledFaceMovement=true - true to move the mouth blendshapes with voice audio - * when MyAvatar.hasScriptedBlendshapes is enabled. * @property {number} rotationRecenterFilterLength - Configures how quickly the avatar root rotates to recenter its facing * direction to match that of the user's torso based on head and hands orientation. A smaller value makes the * recentering happen more quickly. The minimum value is 0.01. @@ -312,7 +306,10 @@ class MyAvatar : public Avatar { * @borrows Avatar.setAttachmentsVariant as setAttachmentsVariant * @borrows Avatar.updateAvatarEntity as updateAvatarEntity * @borrows Avatar.clearAvatarEntity as clearAvatarEntity - * @borrows Avatar.setForceFaceTrackerConnected as setForceFaceTrackerConnected + * @borrows Avatar.hasScriptedBlendshapes as hasScriptedBlendshapes + * @borrows Avatar.hasProceduralBlinkFaceMovement as hasProceduralBlinkFaceMovement + * @borrows Avatar.hasEyeFaceMovement as hasEyeFaceMovement + * @borrows Avatar.hasAudioEnabledFaceMovement as hasAudioEnabledFaceMovement * @borrows Avatar.setSkeletonModelURL as setSkeletonModelURL * @borrows Avatar.getAttachmentData as getAttachmentData * @borrows Avatar.setAttachmentData as setAttachmentData @@ -359,10 +356,6 @@ class MyAvatar : public Avatar { Q_PROPERTY(AudioListenerMode audioListenerModeCustom READ getAudioListenerModeCustom) Q_PROPERTY(glm::vec3 customListenPosition READ getCustomListenPosition WRITE setCustomListenPosition) Q_PROPERTY(glm::quat customListenOrientation READ getCustomListenOrientation WRITE setCustomListenOrientation) - Q_PROPERTY(bool hasScriptedBlendshapes READ getHasScriptedBlendshapes WRITE setHasScriptedBlendshapes) - Q_PROPERTY(bool hasProceduralBlinkFaceMovement READ getHasProceduralBlinkFaceMovement WRITE setHasProceduralBlinkFaceMovement) - Q_PROPERTY(bool hasProceduralEyeFaceMovement READ getHasProceduralEyeFaceMovement WRITE setHasProceduralEyeFaceMovement) - Q_PROPERTY(bool hasAudioEnabledFaceMovement READ getHasAudioEnabledFaceMovement WRITE setHasAudioEnabledFaceMovement) Q_PROPERTY(float rotationRecenterFilterLength READ getRotationRecenterFilterLength WRITE setRotationRecenterFilterLength) Q_PROPERTY(float rotationThreshold READ getRotationThreshold WRITE setRotationThreshold) Q_PROPERTY(bool enableStepResetRotation READ getEnableStepResetRotation WRITE setEnableStepResetRotation) @@ -2555,14 +2548,6 @@ private: virtual bool shouldRenderHead(const RenderArgs* renderArgs) const override; void setShouldRenderLocally(bool shouldRender) { _shouldRender = shouldRender; setEnableMeshVisible(shouldRender); } bool getShouldRenderLocally() const { return _shouldRender; } - void setHasScriptedBlendshapes(bool hasScriptedBlendshapes); - bool getHasScriptedBlendshapes() const override { return _hasScriptedBlendShapes; } - void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasProceduralBlinkFaceMovement() const override { return _headData->getHasProceduralBlinkFaceMovement(); } - void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralEyeFaceMovement() const override { return _headData->getHasProceduralEyeFaceMovement(); } - void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasAudioEnabledFaceMovement() const override { return _headData->getHasAudioEnabledFaceMovement(); } void setRotationRecenterFilterLength(float length); float getRotationRecenterFilterLength() const { return _rotationRecenterFilterLength; } void setRotationThreshold(float angleRadians); diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index e5c8b71ea2..4705d2d765 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -64,7 +64,9 @@ void MyHead::simulate(float deltaTime) { bool eyeLidsTracked = userInputMapper->getActionStateValid(controller::Action::LEFT_EYE_BLINK) && userInputMapper->getActionStateValid(controller::Action::RIGHT_EYE_BLINK); - setFaceTrackerConnected(eyeLidsTracked); + + setHasScriptedBlendshapes(eyeLidsTracked); + if (eyeLidsTracked) { float leftEyeBlink = userInputMapper->getActionState(controller::Action::LEFT_EYE_BLINK); float rightEyeBlink = userInputMapper->getActionState(controller::Action::RIGHT_EYE_BLINK); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 710bfb8d2a..520241d020 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -110,7 +110,7 @@ AvatarData::AvatarData() : _targetScale(1.0f), _handState(0), _keyState(NO_KEY_DOWN), - _forceFaceTrackerConnected(false), + _hasScriptedBlendshapes(false), _headData(NULL), _errorLogExpiry(0), _owningAvatarMixer(), @@ -154,6 +154,32 @@ float AvatarData::getDomainLimitedScale() const { } } + +void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { + if (hasScriptedBlendshapes == _hasScriptedBlendshapes) { + return; + } + if (!hasScriptedBlendshapes) { + // send a forced avatarData update to make sure the script can send neutal blendshapes on unload + // without having to wait for the update loop, make sure _hasScriptedBlendShapes is still true + // before sending the update, or else it won't send the neutal blendshapes to the receiving clients + sendAvatarDataPacket(true); + } + _hasScriptedBlendshapes = hasScriptedBlendshapes; +} + +void AvatarData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { + _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); +} + +void AvatarData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { + _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); +} + +void AvatarData::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { + _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); +} + void AvatarData::setDomainMinimumHeight(float domainMinimumHeight) { _domainMinimumHeight = glm::clamp(domainMinimumHeight, MIN_AVATAR_HEIGHT, MAX_AVATAR_HEIGHT); } @@ -206,8 +232,8 @@ void AvatarData::lazyInitHeadData() const { if (!_headData) { _headData = new HeadData(const_cast(this)); } - if (_forceFaceTrackerConnected) { - _headData->_isFaceTrackerConnected = true; + if (_hasScriptedBlendshapes) { + _headData->_hasScriptedBlendshapes = true; } } @@ -338,7 +364,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent tranlationChangedSince(lastSentTime) || parentInfoChangedSince(lastSentTime)); hasHandControllers = _controllerLeftHandMatrixCache.isValid() || _controllerRightHandMatrixCache.isValid(); - hasFaceTrackerInfo = !dropFaceTracking && (hasFaceTracker() || getHasScriptedBlendshapes()) && + hasFaceTrackerInfo = !dropFaceTracking && getHasScriptedBlendshapes() && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); hasJointData = !sendMinimum; hasJointDefaultPoseFlags = hasJointData; @@ -529,8 +555,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent setAtBit16(flags, HAND_STATE_FINGER_POINTING_BIT); } // face tracker state - if (_headData->_isFaceTrackerConnected) { - setAtBit16(flags, IS_FACE_TRACKER_CONNECTED); + if (_headData->_hasScriptedBlendshapes) { + setAtBit16(flags, HAS_SCRIPTED_BLENDSHAPES); } // eye tracker state if (!_headData->_hasProceduralEyeMovement) { @@ -1150,7 +1176,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { auto newHandState = getSemiNibbleAt(bitItems, HAND_STATE_START_BIT) + (oneAtBit16(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0); - auto newFaceTrackerConnected = oneAtBit16(bitItems, IS_FACE_TRACKER_CONNECTED); + auto newHasScriptedBlendshapes = oneAtBit16(bitItems, HAS_SCRIPTED_BLENDSHAPES); auto newHasntProceduralEyeMovement = oneAtBit16(bitItems, IS_EYE_TRACKER_CONNECTED); auto newHasAudioEnabledFaceMovement = oneAtBit16(bitItems, AUDIO_ENABLED_FACE_MOVEMENT); @@ -1161,7 +1187,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { bool keyStateChanged = (_keyState != newKeyState); bool handStateChanged = (_handState != newHandState); - bool faceStateChanged = (_headData->_isFaceTrackerConnected != newFaceTrackerConnected); + bool faceStateChanged = (_headData->_hasScriptedBlendshapes != newHasScriptedBlendshapes); bool eyeStateChanged = (_headData->_hasProceduralEyeMovement == newHasntProceduralEyeMovement); bool audioEnableFaceMovementChanged = (_headData->getHasAudioEnabledFaceMovement() != newHasAudioEnabledFaceMovement); bool proceduralEyeFaceMovementChanged = (_headData->getHasProceduralEyeFaceMovement() != newHasProceduralEyeFaceMovement); @@ -1174,7 +1200,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _keyState = newKeyState; _handState = newHandState; - _headData->_isFaceTrackerConnected = newFaceTrackerConnected; + _headData->_hasScriptedBlendshapes = newHasScriptedBlendshapes; _headData->setHasProceduralEyeMovement(!newHasntProceduralEyeMovement); _headData->setHasAudioEnabledFaceMovement(newHasAudioEnabledFaceMovement); _headData->setHasProceduralEyeFaceMovement(newHasProceduralEyeFaceMovement); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index df0783ef4b..6bed5fa9db 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -104,12 +104,12 @@ const quint32 AVATAR_MOTION_SCRIPTABLE_BITS = // Procedural Collide with other avatars is enabled 12th bit // Procedural Has Hero Priority is enabled 13th bit -const int KEY_STATE_START_BIT = 0; // 1st and 2nd bits -const int HAND_STATE_START_BIT = 2; // 3rd and 4th bits -const int IS_FACE_TRACKER_CONNECTED = 4; // 5th bit +const int KEY_STATE_START_BIT = 0; // 1st and 2nd bits (UNUSED) +const int HAND_STATE_START_BIT = 2; // 3rd and 4th bits (UNUSED) +const int HAS_SCRIPTED_BLENDSHAPES = 4; // 5th bit const int IS_EYE_TRACKER_CONNECTED = 5; // 6th bit (was CHAT_CIRCLING) const int HAS_REFERENTIAL = 6; // 7th bit -const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit +const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit (UNUSED) const int AUDIO_ENABLED_FACE_MOVEMENT = 8; // 9th bit const int PROCEDURAL_EYE_FACE_MOVEMENT = 9; // 10th bit const int PROCEDURAL_BLINK_FACE_MOVEMENT = 10; // 11th bit @@ -325,7 +325,7 @@ namespace AvatarDataPacket { // variable length structure follows - // only present if IS_FACE_TRACKER_CONNECTED flag is set in AvatarInfo.flags + // only present if HAS_SCRIPTED_BLENDSHAPES flag is set in AvatarInfo.flags PACKED_BEGIN struct FaceTrackerInfo { float leftEyeBlink; float rightEyeBlink; @@ -534,6 +534,19 @@ class AvatarData : public QObject, public SpatiallyNestable { * size in the virtual world. Read-only. * @property {boolean} hasPriority - true if the avatar is in a "hero" zone, false if it isn't. * Read-only. + * @property {boolean} hasScriptedBlendshapes=false - Set this to true before using the {@link MyAvatar.setBlendshape} method, + * after you no longer want scripted control over the blendshapes set to back to false.
NOTE: this property will + * automatically become true if the Controller system has valid facial blendshape actions. + * @property {boolean} hasProceduralBlinkFaceMovement=true - By default avatars will blink automatically by animating facial + * blendshapes. Set this property to false to disable this automatic blinking. This can be useful if you + * wish to fully control the blink facial blendshapes via the {@link MyAvatar.setBlendshape} method. + * @property {boolean} hasProceduralEyeFaceMovement=true - By default the avatar eye facial blendshapes will be adjusted + * automatically as the eyes move. This will prevent the iris is never obscured by the upper or lower lids. Set this + * property to false to disable this automatic movement. This can be useful if you wish to fully control + * the eye blendshapes via the {@link MyAvatar.setBlendshape} method. + * @property {boolean} hasAudioEnabledFaceMovement=true - By default the avatar mouth blendshapes will animate based on + * the microphone audio. Set this property to false to disable that animaiton. This can be useful if you + * wish to fully control the blink facial blendshapes via the {@link MyAvatar.setBlendshape} method. */ Q_PROPERTY(glm::vec3 position READ getWorldPosition WRITE setPositionViaScript) Q_PROPERTY(float scale READ getDomainLimitedScale WRITE setTargetScale) @@ -575,6 +588,11 @@ class AvatarData : public QObject, public SpatiallyNestable { Q_PROPERTY(bool hasPriority READ getHasPriority) + Q_PROPERTY(bool hasScriptedBlendshapes READ getHasScriptedBlendshapes WRITE setHasScriptedBlendshapes) + Q_PROPERTY(bool hasProceduralBlinkFaceMovement READ getHasProceduralBlinkFaceMovement WRITE setHasProceduralBlinkFaceMovement) + Q_PROPERTY(bool hasProceduralEyeFaceMovement READ getHasProceduralEyeFaceMovement WRITE setHasProceduralEyeFaceMovement) + Q_PROPERTY(bool hasAudioEnabledFaceMovement READ getHasAudioEnabledFaceMovement WRITE setHasAudioEnabledFaceMovement) + public: virtual QString getName() const override { return QString("Avatar:") + _displayName; } @@ -684,10 +702,14 @@ public: float getDomainLimitedScale() const; - virtual bool getHasScriptedBlendshapes() const { return false; } - virtual bool getHasProceduralBlinkFaceMovement() const { return true; } - virtual bool getHasProceduralEyeFaceMovement() const { return true; } - virtual bool getHasAudioEnabledFaceMovement() const { return false; } + void setHasScriptedBlendshapes(bool hasScriptedBlendshapes); + bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } + void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); + bool getHasProceduralBlinkFaceMovement() const { return _headData->getHasProceduralBlinkFaceMovement(); } + void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); + bool getHasProceduralEyeFaceMovement() const { return _headData->getHasProceduralEyeFaceMovement(); } + void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); + bool getHasAudioEnabledFaceMovement() const { return _headData->getHasAudioEnabledFaceMovement(); } /**jsdoc * Gets the minimum scale allowed for this avatar in the current domain. @@ -1111,13 +1133,14 @@ public: /**jsdoc * Sets the value of a blendshape to animate your avatar's face. To enable other users to see the resulting animation of - * your avatar's face, use {@link Avatar.setForceFaceTrackerConnected} or {@link MyAvatar.setForceFaceTrackerConnected}. + * your avatar's face, set {@link Avatar.hasScriptedBlendshapes} to true while using this API and back to false when your + * animation is complete. * @function Avatar.setBlendshape * @param {string} name - The name of the blendshape, per the * {@link https://docs.highfidelity.com/create/avatars/avatar-standards.html#blendshapes Avatar Standards}. * @param {number} value - A value between 0.0 and 1.0. * @example Open your avatar's mouth wide. - * MyAvatar.setForceFaceTrackerConnected(true); + * MyAvatar.hasScriptedBlendshapes = true; * MyAvatar.setBlendshape("JawOpen", 1.0); * * // Note: If using from the Avatar API, replace "MyAvatar" with "Avatar". @@ -1163,15 +1186,16 @@ public: */ Q_INVOKABLE virtual void clearAvatarEntity(const QUuid& entityID, bool requiresRemovalFromTree = true); - /**jsdoc + *

Deprecated: This method is deprecated and will be removed.

+ * Use Avatar.hasScriptedBlendshapes property instead. * Enables blendshapes set using {@link Avatar.setBlendshape} or {@link MyAvatar.setBlendshape} to be transmitted to other * users so that they can see the animation of your avatar's face. * @function Avatar.setForceFaceTrackerConnected * @param {boolean} connected - true to enable blendshape changes to be transmitted to other users, * false to disable. */ - Q_INVOKABLE void setForceFaceTrackerConnected(bool connected) { _forceFaceTrackerConnected = connected; } + Q_INVOKABLE void setForceFaceTrackerConnected(bool connected) { setHasScriptedBlendshapes(connected); } // key state void setKeyState(KeyState s) { _keyState = s; } @@ -1660,7 +1684,6 @@ protected: bool faceTrackerInfoChangedSince(quint64 time) const { return true; } // FIXME bool hasParent() const { return !getParentID().isNull(); } - bool hasFaceTracker() const { return _headData ? _headData->_isFaceTrackerConnected : false; } QByteArray packSkeletonData() const; QByteArray packSkeletonModelURL() const; @@ -1693,7 +1716,7 @@ protected: // key state KeyState _keyState; - bool _forceFaceTrackerConnected; + bool _hasScriptedBlendshapes; bool _hasNewJointData { true }; // set in AvatarData, cleared in Avatar mutable HeadData* _headData { nullptr }; diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index c86e534929..0e0dd9887f 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -206,8 +206,7 @@ void HeadData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement } bool HeadData::getHasProceduralBlinkFaceMovement() const { - // return _hasProceduralBlinkFaceMovement; - return _hasProceduralBlinkFaceMovement && !_isFaceTrackerConnected; + return _hasProceduralBlinkFaceMovement; } void HeadData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { @@ -230,6 +229,6 @@ void HeadData::setHasProceduralEyeMovement(bool hasProceduralEyeMovement) { _hasProceduralEyeMovement = hasProceduralEyeMovement; } -void HeadData::setFaceTrackerConnected(bool value) { - _isFaceTrackerConnected = value; +void HeadData::setHasScriptedBlendshapes(bool value) { + _hasScriptedBlendshapes = value; } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index dc5aaf2595..e5dd158505 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -81,8 +81,8 @@ public: bool getHasProceduralEyeMovement() const; void setHasProceduralEyeMovement(bool hasProceduralEyeMovement); - void setFaceTrackerConnected(bool value); - bool getFaceTrackerConnected() const { return _isFaceTrackerConnected; } + void setHasScriptedBlendshapes(bool value); + bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } friend class AvatarData; @@ -103,7 +103,7 @@ protected: bool _hasProceduralEyeFaceMovement { true }; bool _hasProceduralEyeMovement { true }; - bool _isFaceTrackerConnected { false }; + bool _hasScriptedBlendshapes { false }; float _leftEyeBlink { 0.0f }; float _rightEyeBlink { 0.0f }; diff --git a/libraries/shared/src/FaceshiftConstants.cpp b/libraries/shared/src/FaceshiftConstants.cpp index 0d6f718e49..b847837a7a 100644 --- a/libraries/shared/src/FaceshiftConstants.cpp +++ b/libraries/shared/src/FaceshiftConstants.cpp @@ -34,7 +34,7 @@ const char* FACESHIFT_BLENDSHAPES[] = { "JawFwd", "JawLeft", "JawOpen", - "JawChew", + "JawChew", // legacy not in ARKit "JawRight", "MouthLeft", "MouthRight", @@ -48,21 +48,29 @@ const char* FACESHIFT_BLENDSHAPES[] = { "LipsStretch_R", "LipsUpperClose", "LipsLowerClose", - "LipsUpperUp", - "LipsLowerDown", + "LipsUpperUp", // legacy, split in ARKit + "LipsLowerDown", // legacy, split in ARKit "LipsUpperOpen", "LipsLowerOpen", "LipsFunnel", "LipsPucker", "ChinLowerRaise", "ChinUpperRaise", - "Sneer", + "Sneer", // legacy, split in ARKit "Puff", "CheekSquint_L", "CheekSquint_R", "" }; +// new in ARKit +// LipsTogether +// MouthPressLeft +// MouthPressRight +// MouthShrugLower +// MouthShrugUpper +// TongueOut + const int NUM_FACESHIFT_BLENDSHAPES = sizeof(FACESHIFT_BLENDSHAPES) / sizeof(char*); const int EYE_BLINK_L_INDEX = 0; From da0911e01b8a88b630513788601cee15650c31be Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 8 Oct 2019 16:08:29 -0700 Subject: [PATCH 06/33] Delete legacy blendshape code Move FaceshiftConstants to BlendshapeConstants. Delete FaceTracker and DdeFaceTracker classes. Delete old facetracker menu and preferences --- interface/CMakeLists.txt | 4 +- interface/src/Application.cpp | 113 +-- interface/src/Application.h | 9 - interface/src/Menu.cpp | 42 -- interface/src/avatar/MyAvatar.cpp | 73 +- interface/src/avatar/MyAvatar.h | 4 +- interface/src/avatar/MyHead.cpp | 4 +- interface/src/devices/DdeFaceTracker.cpp | 686 ------------------ interface/src/devices/DdeFaceTracker.h | 181 ----- interface/src/ui/AvatarInputs.cpp | 10 - interface/src/ui/AvatarInputs.h | 33 - interface/src/ui/PreferencesDialog.cpp | 17 - libraries/avatars-renderer/CMakeLists.txt | 2 +- .../src/avatars-renderer/Head.cpp | 19 +- libraries/avatars/src/HeadData.cpp | 2 +- libraries/avatars/src/HeadData.h | 2 +- libraries/fbx/src/FBXSerializer.cpp | 2 +- libraries/fbx/src/GLTFSerializer.cpp | 2 +- ...tConstants.cpp => BlendshapeConstants.cpp} | 14 +- libraries/shared/src/BlendshapeConstants.h | 76 ++ libraries/shared/src/FaceshiftConstants.h | 25 - libraries/trackers/CMakeLists.txt | 7 - .../trackers/src/trackers/FaceTracker.cpp | 133 ---- libraries/trackers/src/trackers/FaceTracker.h | 131 ---- libraries/trackers/src/trackers/Logging.cpp | 11 - libraries/trackers/src/trackers/Logging.h | 16 - tools/jsdoc/plugins/hifi.js | 2 - 27 files changed, 110 insertions(+), 1510 deletions(-) delete mode 100644 interface/src/devices/DdeFaceTracker.cpp delete mode 100644 interface/src/devices/DdeFaceTracker.h rename libraries/shared/src/{FaceshiftConstants.cpp => BlendshapeConstants.cpp} (84%) create mode 100644 libraries/shared/src/BlendshapeConstants.h delete mode 100644 libraries/shared/src/FaceshiftConstants.h delete mode 100644 libraries/trackers/CMakeLists.txt delete mode 100644 libraries/trackers/src/trackers/FaceTracker.cpp delete mode 100644 libraries/trackers/src/trackers/FaceTracker.h delete mode 100644 libraries/trackers/src/trackers/Logging.cpp delete mode 100644 libraries/trackers/src/trackers/Logging.h diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index bcd3f269e8..9030666609 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -211,10 +211,10 @@ endif() link_hifi_libraries( shared workload task octree ktx gpu gl procedural graphics graphics-scripting render pointers recording hfm fbx networking material-networking - model-networking model-baker entities avatars trackers + model-networking model-baker entities avatars audio audio-client animation script-engine physics render-utils entities-renderer avatars-renderer ui qml auto-updater midi - controllers plugins image trackers platform + controllers plugins image platform ui-plugins display-plugins input-plugins # Platform specific GL libraries ${PLATFORM_GL_BACKEND} diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 92bc54d43f..677137148c 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -170,7 +170,6 @@ #include "avatar/MyCharacterController.h" #include "CrashRecoveryHandler.h" #include "CrashHandler.h" -#include "devices/DdeFaceTracker.h" #include "DiscoverabilityManager.h" #include "GLCanvas.h" #include "InterfaceDynamicFactory.h" @@ -887,11 +886,6 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); - -#ifdef HAVE_DDE - DependencyManager::set(); -#endif - DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); @@ -1067,7 +1061,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo _lastSendDownstreamAudioStats(usecTimestampNow()), _notifiedPacketVersionMismatchThisDomain(false), _maxOctreePPS(maxOctreePacketsPerSecond.get()), - _lastFaceTrackerUpdate(0), _snapshotSound(nullptr), _sampleSound(nullptr) { @@ -2014,13 +2007,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo this->installEventFilter(this); - -#ifdef HAVE_DDE - auto ddeTracker = DependencyManager::get(); - ddeTracker->init(); - connect(ddeTracker.data(), &FaceTracker::muteToggled, this, &Application::faceTrackerMuteToggled); -#endif - // If launched from Steam, let it handle updates const QString HIFI_NO_UPDATER_COMMAND_LINE_KEY = "--no-updater"; bool noUpdater = arguments().indexOf(HIFI_NO_UPDATER_COMMAND_LINE_KEY) != -1; @@ -2762,9 +2748,6 @@ void Application::cleanupBeforeQuit() { } // Stop third party processes so that they're not left running in the event of a subsequent shutdown crash. -#ifdef HAVE_DDE - DependencyManager::get()->setEnabled(false); -#endif AnimDebugDraw::getInstance().shutdown(); // FIXME: once we move to shared pointer for the INputDevice we shoud remove this naked delete: @@ -2835,10 +2818,6 @@ void Application::cleanupBeforeQuit() { _window->saveGeometry(); // Destroy third party processes after scripts have finished using them. -#ifdef HAVE_DDE - DependencyManager::destroy(); -#endif - DependencyManager::destroy(); // Must be destroyed before TabletScriptingInterface // stop QML @@ -3457,9 +3436,6 @@ void Application::onDesktopRootContextCreated(QQmlContext* surfaceContext) { surfaceContext->setContextProperty("AccountServices", AccountServicesScriptingInterface::getInstance()); surfaceContext->setContextProperty("DialogsManager", _dialogsManagerScriptingInterface); -#ifdef HAVE_DDE - surfaceContext->setContextProperty("FaceTracker", DependencyManager::get().data()); -#endif surfaceContext->setContextProperty("AvatarManager", DependencyManager::get().data()); surfaceContext->setContextProperty("LODManager", DependencyManager::get().data()); surfaceContext->setContextProperty("HMD", DependencyManager::get().data()); @@ -3723,16 +3699,6 @@ void Application::runTests() { runUnitTests(); } -void Application::faceTrackerMuteToggled() { - - QAction* muteAction = Menu::getInstance()->getActionForOption(MenuOption::MuteFaceTracking); - Q_CHECK_PTR(muteAction); - bool isMuted = getSelectedFaceTracker()->isMuted(); - muteAction->setChecked(isMuted); - getSelectedFaceTracker()->setEnabled(!isMuted); - Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setEnabled(!isMuted); -} - void Application::setFieldOfView(float fov) { if (fov != _fieldOfView.get()) { _fieldOfView.set(fov); @@ -5307,43 +5273,6 @@ ivec2 Application::getMouse() const { return getApplicationCompositor().getReticlePosition(); } -FaceTracker* Application::getActiveFaceTracker() { -#ifdef HAVE_DDE - auto dde = DependencyManager::get(); - - if (dde && dde->isActive()) { - return static_cast(dde.data()); - } -#endif - - return nullptr; -} - -FaceTracker* Application::getSelectedFaceTracker() { - FaceTracker* faceTracker = nullptr; -#ifdef HAVE_DDE - if (Menu::getInstance()->isOptionChecked(MenuOption::UseCamera)) { - faceTracker = DependencyManager::get().data(); - } -#endif - return faceTracker; -} - -void Application::setActiveFaceTracker() const { -#ifdef HAVE_DDE - bool isMuted = Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking); - bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::UseCamera); - Menu::getInstance()->getActionForOption(MenuOption::BinaryEyelidControl)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::CoupleEyelids)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::VelocityFilter)->setVisible(isUsingDDE); - Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setVisible(isUsingDDE); - auto ddeTracker = DependencyManager::get(); - ddeTracker->setIsMuted(isMuted); - ddeTracker->setEnabled(isUsingDDE && !isMuted); -#endif -} - bool Application::exportEntities(const QString& filename, const QVector& entityIDs, const glm::vec3* givenOffset) { @@ -5827,8 +5756,7 @@ void Application::updateMyAvatarLookAtPosition() { PerformanceWarning warn(showWarnings, "Application::updateMyAvatarLookAtPosition()"); auto myAvatar = getMyAvatar(); - FaceTracker* faceTracker = getActiveFaceTracker(); - myAvatar->updateLookAtPosition(faceTracker, _myCamera); + myAvatar->updateLookAtPosition(_myCamera); } void Application::updateThreads(float deltaTime) { @@ -6254,37 +6182,6 @@ void Application::update(float deltaTime) { auto myAvatar = getMyAvatar(); { PerformanceTimer perfTimer("devices"); - - FaceTracker* tracker = getSelectedFaceTracker(); - if (tracker && Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking) != tracker->isMuted()) { - tracker->toggleMute(); - } - - tracker = getActiveFaceTracker(); - if (tracker && !tracker->isMuted()) { - tracker->update(deltaTime); - - // Auto-mute microphone after losing face tracking? - if (tracker->isTracking()) { - _lastFaceTrackerUpdate = usecTimestampNow(); - } else { - const quint64 MUTE_MICROPHONE_AFTER_USECS = 5000000; //5 secs - Menu* menu = Menu::getInstance(); - auto audioClient = DependencyManager::get(); - if (menu->isOptionChecked(MenuOption::AutoMuteAudio) && !audioClient->isMuted()) { - if (_lastFaceTrackerUpdate > 0 - && ((usecTimestampNow() - _lastFaceTrackerUpdate) > MUTE_MICROPHONE_AFTER_USECS)) { - audioClient->setMuted(true); - _lastFaceTrackerUpdate = 0; - } - } else { - _lastFaceTrackerUpdate = 0; - } - } - } else { - _lastFaceTrackerUpdate = 0; - } - auto userInputMapper = DependencyManager::get(); controller::HmdAvatarAlignmentType hmdAvatarAlignmentType; @@ -7080,10 +6977,6 @@ void Application::copyDisplayViewFrustum(ViewFrustum& viewOut) const { // feature. However, we still use this to reset face trackers, eye trackers, audio and to optionally re-load the avatar // rig and animations from scratch. void Application::resetSensors(bool andReload) { -#ifdef HAVE_DDE - DependencyManager::get()->reset(); -#endif - _overlayConductor.centerUI(); getActiveDisplayPlugin()->resetSensors(); getMyAvatar()->reset(true, andReload); @@ -7485,10 +7378,6 @@ void Application::registerScriptEngineWithApplicationServices(const ScriptEngine scriptEngine->registerGlobalObject("AccountServices", AccountServicesScriptingInterface::getInstance()); qScriptRegisterMetaType(scriptEngine.data(), DownloadInfoResultToScriptValue, DownloadInfoResultFromScriptValue); -#ifdef HAVE_DDE - scriptEngine->registerGlobalObject("FaceTracker", DependencyManager::get().data()); -#endif - scriptEngine->registerGlobalObject("AvatarManager", DependencyManager::get().data()); scriptEngine->registerGlobalObject("LODManager", DependencyManager::get().data()); diff --git a/interface/src/Application.h b/interface/src/Application.h index e3334d12d6..80b10697b8 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -81,7 +81,6 @@ #include "VisionSqueeze.h" class GLCanvas; -class FaceTracker; class MainWindow; class AssetUpload; class CompositorHelper; @@ -191,9 +190,6 @@ public: ivec2 getMouse() const; - FaceTracker* getActiveFaceTracker(); - FaceTracker* getSelectedFaceTracker(); - ApplicationOverlay& getApplicationOverlay() { return _applicationOverlay; } const ApplicationOverlay& getApplicationOverlay() const { return _applicationOverlay; } CompositorHelper& getApplicationCompositor() const; @@ -423,7 +419,6 @@ public slots: static void packageModel(); void resetSensors(bool andReload = false); - void setActiveFaceTracker() const; void hmdVisibleChanged(bool visible); @@ -497,8 +492,6 @@ private slots: void resettingDomain(); - void faceTrackerMuteToggled(); - void activeChanged(Qt::ApplicationState state); void windowMinimizedChanged(bool minimized); @@ -736,8 +729,6 @@ private: PerformanceManager _performanceManager; RefreshRateManager _refreshRateManager; - quint64 _lastFaceTrackerUpdate; - GameWorkload _gameWorkload; GraphicsEngine _graphicsEngine; diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 5be4db46a0..c0df4ff159 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -37,7 +37,6 @@ #include "avatar/AvatarManager.h" #include "avatar/AvatarPackager.h" #include "AvatarBookmarks.h" -#include "devices/DdeFaceTracker.h" #include "MainWindow.h" #include "render/DrawStatus.h" #include "scripting/MenuScriptingInterface.h" @@ -493,47 +492,6 @@ Menu::Menu() { // Developer > Avatar >>> MenuWrapper* avatarDebugMenu = developerMenu->addMenu("Avatar"); - // Developer > Avatar > Face Tracking - MenuWrapper* faceTrackingMenu = avatarDebugMenu->addMenu("Face Tracking"); - { - QActionGroup* faceTrackerGroup = new QActionGroup(avatarDebugMenu); - - bool defaultNoFaceTracking = true; -#ifdef HAVE_DDE - defaultNoFaceTracking = false; -#endif - QAction* noFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::NoFaceTracking, - 0, defaultNoFaceTracking, - qApp, SLOT(setActiveFaceTracker())); - faceTrackerGroup->addAction(noFaceTracker); - -#ifdef HAVE_DDE - QAction* ddeFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseCamera, - 0, true, - qApp, SLOT(setActiveFaceTracker())); - faceTrackerGroup->addAction(ddeFaceTracker); -#endif - } -#ifdef HAVE_DDE - faceTrackingMenu->addSeparator(); - QAction* binaryEyelidControl = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::BinaryEyelidControl, 0, true); - binaryEyelidControl->setVisible(true); // DDE face tracking is on by default - QAction* coupleEyelids = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::CoupleEyelids, 0, true); - coupleEyelids->setVisible(true); // DDE face tracking is on by default - QAction* useAudioForMouth = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseAudioForMouth, 0, true); - useAudioForMouth->setVisible(true); // DDE face tracking is on by default - QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::VelocityFilter, 0, true); - ddeFiltering->setVisible(true); // DDE face tracking is on by default - QAction* ddeCalibrate = addActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::CalibrateCamera, 0, - DependencyManager::get().data(), SLOT(calibrate())); - ddeCalibrate->setVisible(true); // DDE face tracking is on by default - faceTrackingMenu->addSeparator(); - addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::MuteFaceTracking, - [](bool mute) { FaceTracker::setIsMuted(mute); }, - Qt::CTRL | Qt::SHIFT | Qt::Key_F, FaceTracker::isMuted()); - addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::AutoMuteAudio, 0, false); -#endif - action = addCheckableActionToQMenuAndActionHash(avatarDebugMenu, MenuOption::AvatarReceiveStats, 0, false); connect(action, &QAction::triggered, [this]{ Avatar::setShowReceiveStats(isOptionChecked(MenuOption::AvatarReceiveStats)); }); action = addCheckableActionToQMenuAndActionHash(avatarDebugMenu, MenuOption::ShowBoundingCollisionShapes, 0, false); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 3c27ef52c2..fb42f89048 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -48,7 +48,6 @@ #include #include #include -#include #include #include @@ -749,7 +748,6 @@ void MyAvatar::update(float deltaTime) { Head* head = getHead(); head->relax(deltaTime); - updateFromTrackers(deltaTime); if (getIsInWalkingState() && glm::length(getControllerPoseInAvatarFrame(controller::Action::HEAD).getVelocity()) < DEFAULT_AVATAR_WALK_SPEED_THRESHOLD) { setIsInWalkingState(false); @@ -1140,60 +1138,6 @@ void MyAvatar::updateSensorToWorldMatrix() { } -// Update avatar head rotation with sensor data -void MyAvatar::updateFromTrackers(float deltaTime) { - glm::vec3 estimatedRotation; - - bool hasHead = getControllerPoseInAvatarFrame(controller::Action::HEAD).isValid(); - bool playing = DependencyManager::get()->isPlaying(); - if (hasHead && playing) { - return; - } - - FaceTracker* tracker = qApp->getActiveFaceTracker(); - bool inFacetracker = tracker && !FaceTracker::isMuted(); - - if (inFacetracker) { - estimatedRotation = glm::degrees(safeEulerAngles(tracker->getHeadRotation())); - } - - // Rotate the body if the head is turned beyond the screen - if (Menu::getInstance()->isOptionChecked(MenuOption::TurnWithHead)) { - const float TRACKER_YAW_TURN_SENSITIVITY = 0.5f; - const float TRACKER_MIN_YAW_TURN = 15.0f; - const float TRACKER_MAX_YAW_TURN = 50.0f; - if ( (fabs(estimatedRotation.y) > TRACKER_MIN_YAW_TURN) && - (fabs(estimatedRotation.y) < TRACKER_MAX_YAW_TURN) ) { - if (estimatedRotation.y > 0.0f) { - _bodyYawDelta += (estimatedRotation.y - TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; - } else { - _bodyYawDelta += (estimatedRotation.y + TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; - } - } - } - - // Set the rotation of the avatar's head (as seen by others, not affecting view frustum) - // to be scaled such that when the user's physical head is pointing at edge of screen, the - // avatar head is at the edge of the in-world view frustum. So while a real person may move - // their head only 30 degrees or so, this may correspond to a 90 degree field of view. - // Note that roll is magnified by a constant because it is not related to field of view. - - - Head* head = getHead(); - if (hasHead || playing) { - head->setDeltaPitch(estimatedRotation.x); - head->setDeltaYaw(estimatedRotation.y); - head->setDeltaRoll(estimatedRotation.z); - } else { - ViewFrustum viewFrustum; - qApp->copyViewFrustum(viewFrustum); - float magnifyFieldOfView = viewFrustum.getFieldOfView() / _realWorldFieldOfView.get(); - head->setDeltaPitch(estimatedRotation.x * magnifyFieldOfView); - head->setDeltaYaw(estimatedRotation.y * magnifyFieldOfView); - head->setDeltaRoll(estimatedRotation.z); - } -} - glm::vec3 MyAvatar::getLeftHandPosition() const { auto pose = getControllerPoseInAvatarFrame(controller::Action::LEFT_HAND); return pose.isValid() ? pose.getTranslation() : glm::vec3(0.0f); @@ -6585,7 +6529,7 @@ bool MyAvatar::getIsJointOverridden(int jointIndex) const { return _skeletonModel->getIsJointOverridden(jointIndex); } -void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) { +void MyAvatar::updateLookAtPosition(Camera& myCamera) { updateLookAtTargetAvatar(); @@ -6681,21 +6625,6 @@ void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) (getHead()->getFinalOrientationInWorldFrame() * glm::vec3(0.0f, 0.0f, -TREE_SCALE)); } } - - // Deflect the eyes a bit to match the detected gaze from the face tracker if active. - if (faceTracker && !faceTracker->isMuted()) { - float eyePitch = faceTracker->getEstimatedEyePitch(); - float eyeYaw = faceTracker->getEstimatedEyeYaw(); - const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f; - glm::vec3 origin = myHead->getEyePosition(); - float deflection = faceTracker->getEyeDeflection(); - if (isLookingAtSomeone) { - deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT; - } - lookAtSpot = origin + myCamera.getOrientation() * glm::quat(glm::radians(glm::vec3( - eyePitch * deflection, eyeYaw * deflection, 0.0f))) * - glm::inverse(myCamera.getOrientation()) * (lookAtSpot - origin); - } } } diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index acbd0f27a5..d67a2567f8 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -34,7 +34,6 @@ #include "AtRestDetector.h" #include "MyCharacterController.h" #include "RingBufferHistory.h" -#include "devices/DdeFaceTracker.h" class AvatarActionHold; class ModelItemID; @@ -1899,7 +1898,7 @@ public: bool getFlowActive() const; bool getNetworkGraphActive() const; - void updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera); + void updateLookAtPosition(Camera& myCamera); // sets the reaction enabled and triggered parameters of the passed in params // also clears internal reaction triggers @@ -2542,7 +2541,6 @@ private: virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking) override; void simulate(float deltaTime, bool inView) override; - void updateFromTrackers(float deltaTime); void saveAvatarUrl(); virtual void render(RenderArgs* renderArgs) override; virtual bool shouldRenderHead(const RenderArgs* renderArgs) const override; diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 4705d2d765..1c6e4690dd 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -14,10 +14,8 @@ #include #include #include -#include -#include +#include -#include "devices/DdeFaceTracker.h" #include "Application.h" #include "MyAvatar.h" diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp deleted file mode 100644 index b9dc8326e8..0000000000 --- a/interface/src/devices/DdeFaceTracker.cpp +++ /dev/null @@ -1,686 +0,0 @@ -// -// DdeFaceTracker.cpp -// -// -// Created by Clement on 8/2/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "DdeFaceTracker.h" - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "Application.h" -#include "InterfaceLogging.h" -#include "Menu.h" - - -static const QHostAddress DDE_SERVER_ADDR("127.0.0.1"); -static const quint16 DDE_SERVER_PORT = 64204; -static const quint16 DDE_CONTROL_PORT = 64205; -#if defined(Q_OS_WIN) -static const QString DDE_PROGRAM_PATH = "/dde/dde.exe"; -#elif defined(Q_OS_MAC) -static const QString DDE_PROGRAM_PATH = "/dde.app/Contents/MacOS/dde"; -#endif -static const QStringList DDE_ARGUMENTS = QStringList() - << "--udp=" + DDE_SERVER_ADDR.toString() + ":" + QString::number(DDE_SERVER_PORT) - << "--receiver=" + QString::number(DDE_CONTROL_PORT) - << "--facedet_interval=500" // ms - << "--headless"; - -static const int NUM_EXPRESSIONS = 46; -static const int MIN_PACKET_SIZE = (8 + NUM_EXPRESSIONS) * sizeof(float) + sizeof(int); -static const int MAX_NAME_SIZE = 31; - -// There's almost but not quite a 1-1 correspondence between DDE's 46 and Faceshift 1.3's 48 packets. -// The best guess at mapping is to: -// - Swap L and R values -// - Skip two Faceshift values: JawChew (22) and LipsLowerDown (37) -static const int DDE_TO_FACESHIFT_MAPPING[] = { - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, - 16, - 18, 17, - 19, - 23, - 21, - // Skip JawChew - 20, - 25, 24, 27, 26, 29, 28, 31, 30, 33, 32, - 34, 35, 36, - // Skip LipsLowerDown - 38, 39, 40, 41, 42, 43, 44, 45, - 47, 46 -}; - -// The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much -// less than this. -static const float DDE_COEFFICIENT_SCALES[] = { - 1.0f, // EyeBlink_L - 1.0f, // EyeBlink_R - 1.0f, // EyeSquint_L - 1.0f, // EyeSquint_R - 1.0f, // EyeDown_L - 1.0f, // EyeDown_R - 1.0f, // EyeIn_L - 1.0f, // EyeIn_R - 1.0f, // EyeOpen_L - 1.0f, // EyeOpen_R - 1.0f, // EyeOut_L - 1.0f, // EyeOut_R - 1.0f, // EyeUp_L - 1.0f, // EyeUp_R - 3.0f, // BrowsD_L - 3.0f, // BrowsD_R - 3.0f, // BrowsU_C - 3.0f, // BrowsU_L - 3.0f, // BrowsU_R - 1.0f, // JawFwd - 2.0f, // JawLeft - 1.8f, // JawOpen - 1.0f, // JawChew - 2.0f, // JawRight - 1.5f, // MouthLeft - 1.5f, // MouthRight - 1.5f, // MouthFrown_L - 1.5f, // MouthFrown_R - 2.5f, // MouthSmile_L - 2.5f, // MouthSmile_R - 1.0f, // MouthDimple_L - 1.0f, // MouthDimple_R - 1.0f, // LipsStretch_L - 1.0f, // LipsStretch_R - 1.0f, // LipsUpperClose - 1.0f, // LipsLowerClose - 1.0f, // LipsUpperUp - 1.0f, // LipsLowerDown - 1.0f, // LipsUpperOpen - 1.0f, // LipsLowerOpen - 1.5f, // LipsFunnel - 2.5f, // LipsPucker - 1.5f, // ChinLowerRaise - 1.5f, // ChinUpperRaise - 1.0f, // Sneer - 3.0f, // Puff - 1.0f, // CheekSquint_L - 1.0f // CheekSquint_R -}; - -struct DDEPacket { - //roughly in mm - float focal_length[1]; - float translation[3]; - - //quaternion - float rotation[4]; - - // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much - // less than this. - float expressions[NUM_EXPRESSIONS]; - - //avatar id selected on the UI - int avatar_id; - - //client name, arbitrary length - char name[MAX_NAME_SIZE + 1]; -}; - -static const float STARTING_DDE_MESSAGE_TIME = 0.033f; -static const float DEFAULT_DDE_EYE_CLOSING_THRESHOLD = 0.8f; -static const int CALIBRATION_SAMPLES = 150; - -DdeFaceTracker::DdeFaceTracker() : - DdeFaceTracker(QHostAddress::Any, DDE_SERVER_PORT, DDE_CONTROL_PORT) -{ - -} - -DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, quint16 controlPort) : - _ddeProcess(NULL), - _ddeStopping(false), - _host(host), - _serverPort(serverPort), - _controlPort(controlPort), - _lastReceiveTimestamp(0), - _reset(false), - _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes - _rightBlinkIndex(1), - _leftEyeDownIndex(4), - _rightEyeDownIndex(5), - _leftEyeInIndex(6), - _rightEyeInIndex(7), - _leftEyeOpenIndex(8), - _rightEyeOpenIndex(9), - _browDownLeftIndex(14), - _browDownRightIndex(15), - _browUpCenterIndex(16), - _browUpLeftIndex(17), - _browUpRightIndex(18), - _mouthSmileLeftIndex(28), - _mouthSmileRightIndex(29), - _jawOpenIndex(21), - _lastMessageReceived(0), - _averageMessageTime(STARTING_DDE_MESSAGE_TIME), - _lastHeadTranslation(glm::vec3(0.0f)), - _filteredHeadTranslation(glm::vec3(0.0f)), - _lastBrowUp(0.0f), - _filteredBrowUp(0.0f), - _eyePitch(0.0f), - _eyeYaw(0.0f), - _lastEyePitch(0.0f), - _lastEyeYaw(0.0f), - _filteredEyePitch(0.0f), - _filteredEyeYaw(0.0f), - _longTermAverageEyePitch(0.0f), - _longTermAverageEyeYaw(0.0f), - _lastEyeBlinks(), - _filteredEyeBlinks(), - _lastEyeCoefficients(), - _eyeClosingThreshold("ddeEyeClosingThreshold", DEFAULT_DDE_EYE_CLOSING_THRESHOLD), - _isCalibrating(false), - _calibrationCount(0), - _calibrationValues(), - _calibrationBillboard(NULL), - _calibrationMessage(QString()), - _isCalibrated(false) -{ - _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES); - _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES); - _coefficientAverages.resize(NUM_FACESHIFT_BLENDSHAPES); - _calibrationValues.resize(NUM_FACESHIFT_BLENDSHAPES); - - _eyeStates[0] = EYE_UNCONTROLLED; - _eyeStates[1] = EYE_UNCONTROLLED; - - connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); - connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError))); - connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), - SLOT(socketStateChanged(QAbstractSocket::SocketState))); -} - -DdeFaceTracker::~DdeFaceTracker() { - setEnabled(false); - - if (_isCalibrating) { - cancelCalibration(); - } -} - -void DdeFaceTracker::init() { - FaceTracker::init(); - setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::UseCamera) && !_isMuted); - Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setEnabled(!_isMuted); -} - -void DdeFaceTracker::setEnabled(bool enabled) { - if (!_isInitialized) { - // Don't enable until have explicitly initialized - return; - } -#ifdef HAVE_DDE - - if (_isCalibrating) { - cancelCalibration(); - } - - // isOpen() does not work as one might expect on QUdpSocket; don't test isOpen() before closing socket. - _udpSocket.close(); - - // Terminate any existing DDE process, perhaps left running after an Interface crash. - // Do this even if !enabled in case user reset their settings after crash. - const char* DDE_EXIT_COMMAND = "exit"; - _udpSocket.bind(_host, _serverPort); - _udpSocket.writeDatagram(DDE_EXIT_COMMAND, DDE_SERVER_ADDR, _controlPort); - - if (enabled && !_ddeProcess) { - _ddeStopping = false; - qCDebug(interfaceapp) << "DDE Face Tracker: Starting"; - _ddeProcess = new QProcess(qApp); - connect(_ddeProcess, SIGNAL(finished(int, QProcess::ExitStatus)), SLOT(processFinished(int, QProcess::ExitStatus))); - _ddeProcess->start(QCoreApplication::applicationDirPath() + DDE_PROGRAM_PATH, DDE_ARGUMENTS); - } - - if (!enabled && _ddeProcess) { - _ddeStopping = true; - qCDebug(interfaceapp) << "DDE Face Tracker: Stopping"; - } -#endif -} - -void DdeFaceTracker::processFinished(int exitCode, QProcess::ExitStatus exitStatus) { - if (_ddeProcess) { - if (_ddeStopping) { - qCDebug(interfaceapp) << "DDE Face Tracker: Stopped"; - - } else { - qCWarning(interfaceapp) << "DDE Face Tracker: Stopped unexpectedly"; - Menu::getInstance()->setIsOptionChecked(MenuOption::NoFaceTracking, true); - } - _udpSocket.close(); - delete _ddeProcess; - _ddeProcess = NULL; - } -} - -void DdeFaceTracker::reset() { - if (_udpSocket.state() == QAbstractSocket::BoundState) { - _reset = true; - - qCDebug(interfaceapp) << "DDE Face Tracker: Reset"; - - const char* DDE_RESET_COMMAND = "reset"; - _udpSocket.writeDatagram(DDE_RESET_COMMAND, DDE_SERVER_ADDR, _controlPort); - - FaceTracker::reset(); - - _reset = true; - } -} - -void DdeFaceTracker::update(float deltaTime) { - if (!isActive()) { - return; - } - FaceTracker::update(deltaTime); - - glm::vec3 headEulers = glm::degrees(glm::eulerAngles(_headRotation)); - _estimatedEyePitch = _eyePitch - headEulers.x; - _estimatedEyeYaw = _eyeYaw - headEulers.y; -} - -bool DdeFaceTracker::isActive() const { - return (_ddeProcess != NULL); -} - -bool DdeFaceTracker::isTracking() const { - static const quint64 ACTIVE_TIMEOUT_USECS = 3000000; //3 secs - return (usecTimestampNow() - _lastReceiveTimestamp < ACTIVE_TIMEOUT_USECS); -} - -//private slots and methods -void DdeFaceTracker::socketErrorOccurred(QAbstractSocket::SocketError socketError) { - qCWarning(interfaceapp) << "DDE Face Tracker: Socket error: " << _udpSocket.errorString(); -} - -void DdeFaceTracker::socketStateChanged(QAbstractSocket::SocketState socketState) { - QString state; - switch(socketState) { - case QAbstractSocket::BoundState: - state = "Bound"; - break; - case QAbstractSocket::ClosingState: - state = "Closing"; - break; - case QAbstractSocket::ConnectedState: - state = "Connected"; - break; - case QAbstractSocket::ConnectingState: - state = "Connecting"; - break; - case QAbstractSocket::HostLookupState: - state = "Host Lookup"; - break; - case QAbstractSocket::ListeningState: - state = "Listening"; - break; - case QAbstractSocket::UnconnectedState: - state = "Unconnected"; - break; - } - qCDebug(interfaceapp) << "DDE Face Tracker: Socket: " << state; -} - -void DdeFaceTracker::readPendingDatagrams() { - QByteArray buffer; - while (_udpSocket.hasPendingDatagrams()) { - buffer.resize(_udpSocket.pendingDatagramSize()); - _udpSocket.readDatagram(buffer.data(), buffer.size()); - } - decodePacket(buffer); -} - -float DdeFaceTracker::getBlendshapeCoefficient(int index) const { - return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; -} - -void DdeFaceTracker::decodePacket(const QByteArray& buffer) { - _lastReceiveTimestamp = usecTimestampNow(); - - if (buffer.size() > MIN_PACKET_SIZE) { - if (!_isCalibrated) { - calibrate(); - } - - bool isFiltering = Menu::getInstance()->isOptionChecked(MenuOption::VelocityFilter); - - DDEPacket packet; - int bytesToCopy = glm::min((int)sizeof(packet), buffer.size()); - memset(&packet.name, '\n', MAX_NAME_SIZE + 1); - memcpy(&packet, buffer.data(), bytesToCopy); - - glm::vec3 translation; - memcpy(&translation, packet.translation, sizeof(packet.translation)); - glm::quat rotation; - memcpy(&rotation, &packet.rotation, sizeof(packet.rotation)); - if (_reset || (_lastMessageReceived == 0)) { - memcpy(&_referenceTranslation, &translation, sizeof(glm::vec3)); - memcpy(&_referenceRotation, &rotation, sizeof(glm::quat)); - _reset = false; - } - - // Compute relative translation - float LEAN_DAMPING_FACTOR = 75.0f; - translation -= _referenceTranslation; - translation /= LEAN_DAMPING_FACTOR; - translation.x *= -1; - if (isFiltering) { - glm::vec3 linearVelocity = (translation - _lastHeadTranslation) / _averageMessageTime; - const float LINEAR_VELOCITY_FILTER_STRENGTH = 0.3f; - float velocityFilter = glm::clamp(1.0f - glm::length(linearVelocity) * - LINEAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredHeadTranslation = velocityFilter * _filteredHeadTranslation + (1.0f - velocityFilter) * translation; - _lastHeadTranslation = translation; - _headTranslation = _filteredHeadTranslation; - } else { - _headTranslation = translation; - } - - // Compute relative rotation - rotation = glm::inverse(_referenceRotation) * rotation; - if (isFiltering) { - glm::quat r = glm::normalize(rotation * glm::inverse(_headRotation)); - float theta = 2 * acos(r.w); - glm::vec3 angularVelocity; - if (theta > EPSILON) { - float rMag = glm::length(glm::vec3(r.x, r.y, r.z)); - angularVelocity = theta / _averageMessageTime * glm::vec3(r.x, r.y, r.z) / rMag; - } else { - angularVelocity = glm::vec3(0, 0, 0); - } - const float ANGULAR_VELOCITY_FILTER_STRENGTH = 0.3f; - _headRotation = safeMix(_headRotation, rotation, glm::clamp(glm::length(angularVelocity) * - ANGULAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f)); - } else { - _headRotation = rotation; - } - - // Translate DDE coefficients to Faceshift compatible coefficients - for (int i = 0; i < NUM_EXPRESSIONS; i++) { - _coefficients[DDE_TO_FACESHIFT_MAPPING[i]] = packet.expressions[i]; - } - - // Calibration - if (_isCalibrating) { - addCalibrationDatum(); - } - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _coefficients[i] -= _coefficientAverages[i]; - } - - // Use BrowsU_C to control both brows' up and down - float browUp = _coefficients[_browUpCenterIndex]; - if (isFiltering) { - const float BROW_VELOCITY_FILTER_STRENGTH = 0.5f; - float velocity = fabsf(browUp - _lastBrowUp) / _averageMessageTime; - float velocityFilter = glm::clamp(velocity * BROW_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredBrowUp = velocityFilter * browUp + (1.0f - velocityFilter) * _filteredBrowUp; - _lastBrowUp = browUp; - browUp = _filteredBrowUp; - _coefficients[_browUpCenterIndex] = browUp; - } - _coefficients[_browUpLeftIndex] = browUp; - _coefficients[_browUpRightIndex] = browUp; - _coefficients[_browDownLeftIndex] = -browUp; - _coefficients[_browDownRightIndex] = -browUp; - - // Offset jaw open coefficient - static const float JAW_OPEN_THRESHOLD = 0.1f; - _coefficients[_jawOpenIndex] = _coefficients[_jawOpenIndex] - JAW_OPEN_THRESHOLD; - - // Offset smile coefficients - static const float SMILE_THRESHOLD = 0.5f; - _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD; - _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD; - - // Eye pitch and yaw - // EyeDown coefficients work better over both +ve and -ve values than EyeUp values. - // EyeIn coefficients work better over both +ve and -ve values than EyeOut values. - // Pitch and yaw values are relative to the screen. - const float EYE_PITCH_SCALE = -1500.0f; // Sign, scale, and average to be similar to Faceshift values. - _eyePitch = EYE_PITCH_SCALE * (_coefficients[_leftEyeDownIndex] + _coefficients[_rightEyeDownIndex]); - const float EYE_YAW_SCALE = 2000.0f; // Scale and average to be similar to Faceshift values. - _eyeYaw = EYE_YAW_SCALE * (_coefficients[_leftEyeInIndex] + _coefficients[_rightEyeInIndex]); - if (isFiltering) { - const float EYE_VELOCITY_FILTER_STRENGTH = 0.005f; - float pitchVelocity = fabsf(_eyePitch - _lastEyePitch) / _averageMessageTime; - float pitchVelocityFilter = glm::clamp(pitchVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredEyePitch = pitchVelocityFilter * _eyePitch + (1.0f - pitchVelocityFilter) * _filteredEyePitch; - _lastEyePitch = _eyePitch; - _eyePitch = _filteredEyePitch; - float yawVelocity = fabsf(_eyeYaw - _lastEyeYaw) / _averageMessageTime; - float yawVelocityFilter = glm::clamp(yawVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredEyeYaw = yawVelocityFilter * _eyeYaw + (1.0f - yawVelocityFilter) * _filteredEyeYaw; - _lastEyeYaw = _eyeYaw; - _eyeYaw = _filteredEyeYaw; - } - - // Velocity filter EyeBlink values - const float DDE_EYEBLINK_SCALE = 3.0f; - float eyeBlinks[] = { DDE_EYEBLINK_SCALE * _coefficients[_leftBlinkIndex], - DDE_EYEBLINK_SCALE * _coefficients[_rightBlinkIndex] }; - if (isFiltering) { - const float BLINK_VELOCITY_FILTER_STRENGTH = 0.3f; - for (int i = 0; i < 2; i++) { - float velocity = fabsf(eyeBlinks[i] - _lastEyeBlinks[i]) / _averageMessageTime; - float velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredEyeBlinks[i] = velocityFilter * eyeBlinks[i] + (1.0f - velocityFilter) * _filteredEyeBlinks[i]; - _lastEyeBlinks[i] = eyeBlinks[i]; - } - } - - // Finesse EyeBlink values - float eyeCoefficients[2]; - if (Menu::getInstance()->isOptionChecked(MenuOption::BinaryEyelidControl)) { - if (_eyeStates[0] == EYE_UNCONTROLLED) { - _eyeStates[0] = EYE_OPEN; - _eyeStates[1] = EYE_OPEN; - } - - for (int i = 0; i < 2; i++) { - // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen - // -ve values control EyeOpen; +ve values control EyeBlink - static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value - eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD); - - // Change to closing or opening states - const float EYE_CONTROL_HYSTERISIS = 0.25f; - float eyeClosingThreshold = getEyeClosingThreshold(); - float eyeOpeningThreshold = eyeClosingThreshold - EYE_CONTROL_HYSTERISIS; - if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > eyeClosingThreshold) { - _eyeStates[i] = EYE_CLOSING; - } else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING) - && eyeCoefficients[i] < eyeOpeningThreshold) { - _eyeStates[i] = EYE_OPENING; - } - - const float EYELID_MOVEMENT_RATE = 10.0f; // units/second - const float EYE_OPEN_SCALE = 0.2f; - if (_eyeStates[i] == EYE_CLOSING) { - // Close eyelid until it's fully closed - float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime; - if (closingValue >= 1.0f) { - _eyeStates[i] = EYE_CLOSED; - eyeCoefficients[i] = 1.0f; - } else { - eyeCoefficients[i] = closingValue; - } - } else if (_eyeStates[i] == EYE_OPENING) { - // Open eyelid until it meets the current adjusted value - float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime; - if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) { - _eyeStates[i] = EYE_OPEN; - eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else { - eyeCoefficients[i] = openingValue; - } - } else if (_eyeStates[i] == EYE_OPEN) { - // Reduce eyelid movement - eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else if (_eyeStates[i] == EYE_CLOSED) { - // Keep eyelid fully closed - eyeCoefficients[i] = 1.0; - } - } - - if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) { - // Couple eyelids - eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f; - } - - _lastEyeCoefficients[0] = eyeCoefficients[0]; - _lastEyeCoefficients[1] = eyeCoefficients[1]; - } else { - _eyeStates[0] = EYE_UNCONTROLLED; - _eyeStates[1] = EYE_UNCONTROLLED; - - eyeCoefficients[0] = _filteredEyeBlinks[0]; - eyeCoefficients[1] = _filteredEyeBlinks[1]; - } - - // Couple eyelid values if configured - use the most "open" value for both - if (Menu::getInstance()->isOptionChecked(MenuOption::CoupleEyelids)) { - float eyeCoefficient = std::min(eyeCoefficients[0], eyeCoefficients[1]); - eyeCoefficients[0] = eyeCoefficient; - eyeCoefficients[1] = eyeCoefficient; - } - - // Use EyeBlink values to control both EyeBlink and EyeOpen - if (eyeCoefficients[0] > 0) { - _coefficients[_leftBlinkIndex] = eyeCoefficients[0]; - _coefficients[_leftEyeOpenIndex] = 0.0f; - } else { - _coefficients[_leftBlinkIndex] = 0.0f; - _coefficients[_leftEyeOpenIndex] = -eyeCoefficients[0]; - } - if (eyeCoefficients[1] > 0) { - _coefficients[_rightBlinkIndex] = eyeCoefficients[1]; - _coefficients[_rightEyeOpenIndex] = 0.0f; - } else { - _coefficients[_rightBlinkIndex] = 0.0f; - _coefficients[_rightEyeOpenIndex] = -eyeCoefficients[1]; - } - - // Scale all coefficients - for (int i = 0; i < NUM_EXPRESSIONS; i++) { - _blendshapeCoefficients[i] - = glm::clamp(DDE_COEFFICIENT_SCALES[i] * _coefficients[i], 0.0f, 1.0f); - } - - // Calculate average frame time - const float FRAME_AVERAGING_FACTOR = 0.99f; - quint64 usecsNow = usecTimestampNow(); - if (_lastMessageReceived != 0) { - _averageMessageTime = FRAME_AVERAGING_FACTOR * _averageMessageTime - + (1.0f - FRAME_AVERAGING_FACTOR) * (float)(usecsNow - _lastMessageReceived) / 1000000.0f; - } - _lastMessageReceived = usecsNow; - - FaceTracker::countFrame(); - - } else { - qCWarning(interfaceapp) << "DDE Face Tracker: Decode error"; - } - - if (_isCalibrating && _calibrationCount > CALIBRATION_SAMPLES) { - finishCalibration(); - } -} - -void DdeFaceTracker::setEyeClosingThreshold(float eyeClosingThreshold) { - _eyeClosingThreshold.set(eyeClosingThreshold); -} - -static const int CALIBRATION_BILLBOARD_WIDTH = 300; -static const int CALIBRATION_BILLBOARD_HEIGHT = 120; -static QString CALIBRATION_INSTRUCTION_MESSAGE = "Hold still to calibrate camera"; - -void DdeFaceTracker::calibrate() { - if (!Menu::getInstance()->isOptionChecked(MenuOption::UseCamera) || _isMuted) { - return; - } - - if (!_isCalibrating) { - qCDebug(interfaceapp) << "DDE Face Tracker: Calibration started"; - - _isCalibrating = true; - _calibrationCount = 0; - _calibrationMessage = CALIBRATION_INSTRUCTION_MESSAGE + "\n\n"; - - // FIXME: this overlay probably doesn't work anymore - _calibrationBillboard = new TextOverlay(); - glm::vec2 viewport = qApp->getCanvasSize(); - _calibrationBillboard->setX((viewport.x - CALIBRATION_BILLBOARD_WIDTH) / 2); - _calibrationBillboard->setY((viewport.y - CALIBRATION_BILLBOARD_HEIGHT) / 2); - _calibrationBillboard->setWidth(CALIBRATION_BILLBOARD_WIDTH); - _calibrationBillboard->setHeight(CALIBRATION_BILLBOARD_HEIGHT); - _calibrationBillboardID = qApp->getOverlays().addOverlay(_calibrationBillboard); - - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _calibrationValues[i] = 0.0f; - } - } -} - -void DdeFaceTracker::addCalibrationDatum() { - const int LARGE_TICK_INTERVAL = 30; - const int SMALL_TICK_INTERVAL = 6; - int samplesLeft = CALIBRATION_SAMPLES - _calibrationCount; - if (samplesLeft % LARGE_TICK_INTERVAL == 0) { - _calibrationMessage += QString::number(samplesLeft / LARGE_TICK_INTERVAL); - // FIXME: set overlay text - } else if (samplesLeft % SMALL_TICK_INTERVAL == 0) { - _calibrationMessage += "."; - // FIXME: set overlay text - } - - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _calibrationValues[i] += _coefficients[i]; - } - - _calibrationCount += 1; -} - -void DdeFaceTracker::cancelCalibration() { - qApp->getOverlays().deleteOverlay(_calibrationBillboardID); - _calibrationBillboard = NULL; - _isCalibrating = false; - qCDebug(interfaceapp) << "DDE Face Tracker: Calibration cancelled"; -} - -void DdeFaceTracker::finishCalibration() { - qApp->getOverlays().deleteOverlay(_calibrationBillboardID); - _calibrationBillboard = NULL; - _isCalibrating = false; - _isCalibrated = true; - - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - _coefficientAverages[i] = _calibrationValues[i] / (float)CALIBRATION_SAMPLES; - } - - reset(); - - qCDebug(interfaceapp) << "DDE Face Tracker: Calibration finished"; -} diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h deleted file mode 100644 index dc451134f0..0000000000 --- a/interface/src/devices/DdeFaceTracker.h +++ /dev/null @@ -1,181 +0,0 @@ -// -// DdeFaceTracker.h -// -// -// Created by Clement on 8/2/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_DdeFaceTracker_h -#define hifi_DdeFaceTracker_h - -#include - -//Disabling dde due to random crashes with closing the socket on macos. all the accompanying code is wrapped with the ifdef HAVE_DDE. uncomment the define below to enable -#if defined(Q_OS_WIN) || defined(Q_OS_OSX) - //#define HAVE_DDE -#endif - -#include -#include - -#include -#include - -#include - -/**jsdoc - * The FaceTracker API helps manage facial tracking hardware. - * @namespace FaceTracker - * - * @hifi-interface - * @hifi-client-entity - * @hifi-avatar - */ - -class DdeFaceTracker : public FaceTracker, public Dependency { - Q_OBJECT - SINGLETON_DEPENDENCY - -public: - virtual void init() override; - virtual void reset() override; - virtual void update(float deltaTime) override; - - virtual bool isActive() const override; - virtual bool isTracking() const override; - - float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } - float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } - float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } - float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); } - - float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); } - float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); } - float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); } - float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); } - float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); } - - float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); } - float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } - float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } - - float getEyeClosingThreshold() { return _eyeClosingThreshold.get(); } - void setEyeClosingThreshold(float eyeClosingThreshold); - -public slots: - - /**jsdoc - * @function FaceTracker.setEnabled - * @param {boolean} enabled - */ - void setEnabled(bool enabled) override; - - /**jsdoc - * @function FaceTracker.calibrate - */ - void calibrate(); - -private slots: - void processFinished(int exitCode, QProcess::ExitStatus exitStatus); - - //sockets - void socketErrorOccurred(QAbstractSocket::SocketError socketError); - void readPendingDatagrams(); - void socketStateChanged(QAbstractSocket::SocketState socketState); - -private: - DdeFaceTracker(); - DdeFaceTracker(const QHostAddress& host, quint16 serverPort, quint16 controlPort); - virtual ~DdeFaceTracker(); - - QProcess* _ddeProcess; - bool _ddeStopping; - - QHostAddress _host; - quint16 _serverPort; - quint16 _controlPort; - - float getBlendshapeCoefficient(int index) const; - void decodePacket(const QByteArray& buffer); - - // sockets - QUdpSocket _udpSocket; - quint64 _lastReceiveTimestamp; - - bool _reset; - glm::vec3 _referenceTranslation; - glm::quat _referenceRotation; - - int _leftBlinkIndex; - int _rightBlinkIndex; - int _leftEyeDownIndex; - int _rightEyeDownIndex; - int _leftEyeInIndex; - int _rightEyeInIndex; - int _leftEyeOpenIndex; - int _rightEyeOpenIndex; - - int _browDownLeftIndex; - int _browDownRightIndex; - int _browUpCenterIndex; - int _browUpLeftIndex; - int _browUpRightIndex; - - int _mouthSmileLeftIndex; - int _mouthSmileRightIndex; - - int _jawOpenIndex; - - QVector _coefficients; - - quint64 _lastMessageReceived; - float _averageMessageTime; - - glm::vec3 _lastHeadTranslation; - glm::vec3 _filteredHeadTranslation; - - float _lastBrowUp; - float _filteredBrowUp; - - float _eyePitch; // Degrees, relative to screen - float _eyeYaw; - float _lastEyePitch; - float _lastEyeYaw; - float _filteredEyePitch; - float _filteredEyeYaw; - float _longTermAverageEyePitch = 0.0f; - float _longTermAverageEyeYaw = 0.0f; - bool _longTermAverageInitialized = false; - - enum EyeState { - EYE_UNCONTROLLED, - EYE_OPEN, - EYE_CLOSING, - EYE_CLOSED, - EYE_OPENING - }; - EyeState _eyeStates[2]; - float _lastEyeBlinks[2]; - float _filteredEyeBlinks[2]; - float _lastEyeCoefficients[2]; - Setting::Handle _eyeClosingThreshold; - - QVector _coefficientAverages; - - bool _isCalibrating; - int _calibrationCount; - QVector _calibrationValues; - TextOverlay* _calibrationBillboard; - QUuid _calibrationBillboardID; - QString _calibrationMessage; - bool _isCalibrated; - void addCalibrationDatum(); - void cancelCalibration(); - void finishCalibration(); -}; - -#endif // hifi_DdeFaceTracker_h diff --git a/interface/src/ui/AvatarInputs.cpp b/interface/src/ui/AvatarInputs.cpp index a6bc7cf84f..3986cb1533 100644 --- a/interface/src/ui/AvatarInputs.cpp +++ b/interface/src/ui/AvatarInputs.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include "Application.h" @@ -76,8 +75,6 @@ void AvatarInputs::update() { return; } - AI_UPDATE(cameraEnabled, !Menu::getInstance()->isOptionChecked(MenuOption::NoFaceTracking)); - AI_UPDATE(cameraMuted, Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking)); AI_UPDATE(isHMD, qApp->isHMDMode()); } @@ -103,13 +100,6 @@ bool AvatarInputs::getIgnoreRadiusEnabled() const { return DependencyManager::get()->getIgnoreRadiusEnabled(); } -void AvatarInputs::toggleCameraMute() { - FaceTracker* faceTracker = qApp->getSelectedFaceTracker(); - if (faceTracker) { - faceTracker->toggleMute(); - } -} - void AvatarInputs::resetSensors() { qApp->resetSensors(); } diff --git a/interface/src/ui/AvatarInputs.h b/interface/src/ui/AvatarInputs.h index dca39fd433..40ee0ea7d4 100644 --- a/interface/src/ui/AvatarInputs.h +++ b/interface/src/ui/AvatarInputs.h @@ -32,14 +32,6 @@ class AvatarInputs : public QObject { * @hifi-client-entity * @hifi-avatar * - * @property {boolean} cameraEnabled - true if webcam face tracking is enabled, false if it is - * disabled. - * Read-only. - *

Deprecated: This property is deprecated and will be removed.

- * @property {boolean} cameraMuted - true if webcam face tracking is muted (temporarily disabled), - * false it if isn't. - * Read-only. - *

Deprecated: This property is deprecated and will be removed.

* @property {boolean} ignoreRadiusEnabled - true if the privacy shield is enabled, false if it * is disabled. * Read-only. @@ -51,8 +43,6 @@ class AvatarInputs : public QObject { * it is hidden. */ - AI_PROPERTY(bool, cameraEnabled, false) - AI_PROPERTY(bool, cameraMuted, false) AI_PROPERTY(bool, isHMD, false) Q_PROPERTY(bool showAudioTools READ showAudioTools WRITE setShowAudioTools NOTIFY showAudioToolsChanged) @@ -97,22 +87,6 @@ public slots: signals: - /**jsdoc - * Triggered when webcam face tracking is enabled or disabled. - * @deprecated This signal is deprecated and will be removed. - * @function AvatarInputs.cameraEnabledChanged - * @returns {Signal} - */ - void cameraEnabledChanged(); - - /**jsdoc - * Triggered when webcam face tracking is muted (temporarily disabled) or unmuted. - * @deprecated This signal is deprecated and will be removed. - * @function AvatarInputs.cameraMutedChanged - * @returns {Signal} - */ - void cameraMutedChanged(); - /**jsdoc * Triggered when the display mode changes between desktop and HMD. * @function AvatarInputs.isHMDChanged @@ -183,13 +157,6 @@ protected: */ Q_INVOKABLE void resetSensors(); - /**jsdoc - * Toggles the muting (temporary disablement) of webcam face tracking on/off. - *

Deprecated: This function is deprecated and will be removed.

- * @function AvatarInputs.toggleCameraMute - */ - Q_INVOKABLE void toggleCameraMute(); - private: void onAvatarEnteredIgnoreRadius(); void onAvatarLeftIgnoreRadius(); diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index ec15dd8111..5f033fe008 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -10,7 +10,6 @@ #include #include -#include #include #include #include @@ -285,22 +284,6 @@ void setupPreferences() { preferences->addPreference(preference); } - static const QString FACE_TRACKING{ "Face Tracking" }; - { -#ifdef HAVE_DDE - auto getter = []()->float { return DependencyManager::get()->getEyeClosingThreshold(); }; - auto setter = [](float value) { DependencyManager::get()->setEyeClosingThreshold(value); }; - preferences->addPreference(new SliderPreference(FACE_TRACKING, "Eye Closing Threshold", getter, setter)); -#endif - } - - - { - auto getter = []()->float { return FaceTracker::getEyeDeflection(); }; - auto setter = [](float value) { FaceTracker::setEyeDeflection(value); }; - preferences->addPreference(new SliderPreference(FACE_TRACKING, "Eye Deflection", getter, setter)); - } - static const QString VR_MOVEMENT{ "VR Movement" }; { auto getter = [myAvatar]()->bool { return myAvatar->getAllowTeleporting(); }; diff --git a/libraries/avatars-renderer/CMakeLists.txt b/libraries/avatars-renderer/CMakeLists.txt index de1ac1a7c2..7929be8b04 100644 --- a/libraries/avatars-renderer/CMakeLists.txt +++ b/libraries/avatars-renderer/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET_NAME avatars-renderer) setup_hifi_library(Network Script) -link_hifi_libraries(shared shaders gpu graphics animation material-networking model-networking script-engine render render-utils image trackers entities-renderer) +link_hifi_libraries(shared shaders gpu graphics animation material-networking model-networking script-engine render render-utils image entities-renderer) include_hifi_library_headers(avatars) include_hifi_library_headers(networking) include_hifi_library_headers(hfm) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 63d8e2981c..ea6cdd7087 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include "Logging.h" @@ -26,6 +25,22 @@ using namespace std; static bool disableEyelidAdjustment { false }; +static void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, + float jawOpen, float mouth2, float mouth3, float mouth4, QVector& coefficients) { + + coefficients.resize(std::max((int)coefficients.size(), (int)Blendshapes::BlendshapeCount)); + qFill(coefficients.begin(), coefficients.end(), 0.0f); + coefficients[(int)Blendshapes::EyeBlink_L] = leftBlink; + coefficients[(int)Blendshapes::EyeBlink_R] = rightBlink; + coefficients[(int)Blendshapes::BrowsU_C] = browUp; + coefficients[(int)Blendshapes::BrowsU_L] = browUp; + coefficients[(int)Blendshapes::BrowsU_R] = browUp; + coefficients[(int)Blendshapes::JawOpen] = jawOpen; + coefficients[(int)Blendshapes::MouthSmile_L] = coefficients[(int)Blendshapes::MouthSmile_R] = mouth4; + coefficients[(int)Blendshapes::LipsUpperClose] = mouth2; + coefficients[(int)Blendshapes::LipsFunnel] = mouth3; +} + Head::Head(Avatar* owningAvatar) : HeadData(owningAvatar), _leftEyeLookAtID(DependencyManager::get()->allocateID()), @@ -153,7 +168,7 @@ void Head::simulate(float deltaTime) { _mouthTime = 0.0f; } - FaceTracker::updateFakeCoefficients( + updateFakeCoefficients( _leftEyeBlink, _rightEyeBlink, _browAudioLift, diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 0e0dd9887f..561f2c798a 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -71,7 +71,7 @@ void HeadData::setOrientation(const glm::quat& orientation) { } void HeadData::computeBlendshapesLookupMap(){ - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { _blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i; } } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index e5dd158505..c6bcc2599b 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -20,7 +20,7 @@ #include #include -#include +#include // degrees const float MIN_HEAD_YAW = -180.0f; diff --git a/libraries/fbx/src/FBXSerializer.cpp b/libraries/fbx/src/FBXSerializer.cpp index f8339ddd31..100f6ee98e 100644 --- a/libraries/fbx/src/FBXSerializer.cpp +++ b/libraries/fbx/src/FBXSerializer.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp index da21c7995e..5d4daf53f7 100755 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include "FBXSerializer.h" diff --git a/libraries/shared/src/FaceshiftConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp similarity index 84% rename from libraries/shared/src/FaceshiftConstants.cpp rename to libraries/shared/src/BlendshapeConstants.cpp index b847837a7a..528b941b73 100644 --- a/libraries/shared/src/FaceshiftConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -1,5 +1,5 @@ // -// FaceshiftConstants.cpp +// BlendshapeConstants.cpp // // // Created by Clement on 1/23/15. @@ -9,7 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include "FaceshiftConstants.h" +#include "BlendshapeConstants.h" const char* FACESHIFT_BLENDSHAPES[] = { "EyeBlink_L", @@ -34,7 +34,7 @@ const char* FACESHIFT_BLENDSHAPES[] = { "JawFwd", "JawLeft", "JawOpen", - "JawChew", // legacy not in ARKit + "JawChew", "JawRight", "MouthLeft", "MouthRight", @@ -48,15 +48,15 @@ const char* FACESHIFT_BLENDSHAPES[] = { "LipsStretch_R", "LipsUpperClose", "LipsLowerClose", - "LipsUpperUp", // legacy, split in ARKit - "LipsLowerDown", // legacy, split in ARKit + "LipsUpperUp", + "LipsLowerDown", "LipsUpperOpen", "LipsLowerOpen", "LipsFunnel", "LipsPucker", "ChinLowerRaise", "ChinUpperRaise", - "Sneer", // legacy, split in ARKit + "Sneer", "Puff", "CheekSquint_L", "CheekSquint_R", @@ -71,8 +71,6 @@ const char* FACESHIFT_BLENDSHAPES[] = { // MouthShrugUpper // TongueOut -const int NUM_FACESHIFT_BLENDSHAPES = sizeof(FACESHIFT_BLENDSHAPES) / sizeof(char*); - const int EYE_BLINK_L_INDEX = 0; const int EYE_BLINK_R_INDEX = 1; const int EYE_SQUINT_L_INDEX = 2; diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h new file mode 100644 index 0000000000..0fca03e34b --- /dev/null +++ b/libraries/shared/src/BlendshapeConstants.h @@ -0,0 +1,76 @@ +// +// BlendshapeConstants.h +// +// +// Created by Clement on 1/23/15. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_BlendshapeConstants_h +#define hifi_BlendshapeConstants_h + +/// The names of the blendshapes expected by Faceshift, terminated with an empty string. +extern const char* FACESHIFT_BLENDSHAPES[]; + +// Eyes and Brows indices +extern const int EYE_BLINK_INDICES[]; +extern const int EYE_OPEN_INDICES[]; +extern const int BROWS_U_INDICES[]; +extern const int EYE_SQUINT_INDICES[]; + +enum class Blendshapes : int { + EyeBlink_L = 0, + EyeBlink_R, + EyeSquint_L, + EyeSquint_R, + EyeDown_L, + EyeDown_R, + EyeIn_L, + EyeIn_R, + EyeOpen_L, + EyeOpen_R, + EyeOut_L, // 10 + EyeOut_R, + EyeUp_L, + EyeUp_R, + BrowsD_L, + BrowsD_R, + BrowsU_C, + BrowsU_L, + BrowsU_R, + JawFwd, + JawLeft, // 20 + JawOpen, + JawChew, // legacy not in ARKit + JawRight, + MouthLeft, + MouthRight, + MouthFrown_L, + MouthFrown_R, + MouthSmile_L, + MouthSmile_R, + MouthDimple_L, // 30 + MouthDimple_R, + LipsStretch_L, + LipsStretch_R, + LipsUpperClose, + LipsLowerClose, + LipsUpperUp, // legacy, split in ARKit + LipsLowerDown, // legacy, split in ARKit + LipsUpperOpen, + LipsLowerOpen, + LipsFunnel, // 40 + LipsPucker, + ChinLowerRaise, + ChinUpperRaise, + Sneer, // legacy, split in ARKit + Puff, + CheekSquint_L, + CheekSquint_R, + BlendshapeCount +}; + +#endif // hifi_BlendshapeConstants_h diff --git a/libraries/shared/src/FaceshiftConstants.h b/libraries/shared/src/FaceshiftConstants.h deleted file mode 100644 index 4349a3a21e..0000000000 --- a/libraries/shared/src/FaceshiftConstants.h +++ /dev/null @@ -1,25 +0,0 @@ -// -// FaceshiftConstants.h -// -// -// Created by Clement on 1/23/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_FaceshiftConstants_h -#define hifi_FaceshiftConstants_h - -/// The names of the blendshapes expected by Faceshift, terminated with an empty string. -extern const char* FACESHIFT_BLENDSHAPES[]; -/// The size of FACESHIFT_BLENDSHAPES -extern const int NUM_FACESHIFT_BLENDSHAPES; -// Eyes and Brows indices -extern const int EYE_BLINK_INDICES[]; -extern const int EYE_OPEN_INDICES[]; -extern const int BROWS_U_INDICES[]; -extern const int EYE_SQUINT_INDICES[]; - -#endif // hifi_FaceshiftConstants_h diff --git a/libraries/trackers/CMakeLists.txt b/libraries/trackers/CMakeLists.txt deleted file mode 100644 index 6943f1a197..0000000000 --- a/libraries/trackers/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(TARGET_NAME trackers) -setup_hifi_library() -GroupSources("src") -link_hifi_libraries(shared) -include_hifi_library_headers(octree) - -target_bullet() diff --git a/libraries/trackers/src/trackers/FaceTracker.cpp b/libraries/trackers/src/trackers/FaceTracker.cpp deleted file mode 100644 index 034787f19a..0000000000 --- a/libraries/trackers/src/trackers/FaceTracker.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// -// Created by Andrzej Kapolka on 4/9/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "FaceTracker.h" - -#include -#include -#include "Logging.h" -//#include "Menu.h" - -const int FPS_TIMER_DELAY = 2000; // ms -const int FPS_TIMER_DURATION = 2000; // ms - -const float DEFAULT_EYE_DEFLECTION = 0.25f; -Setting::Handle FaceTracker::_eyeDeflection("faceshiftEyeDeflection", DEFAULT_EYE_DEFLECTION); -bool FaceTracker::_isMuted { true }; - -void FaceTracker::init() { - _isInitialized = true; // FaceTracker can be used now -} - -inline float FaceTracker::getBlendshapeCoefficient(int index) const { - return isValidBlendshapeIndex(index) ? glm::mix(0.0f, _blendshapeCoefficients[index], getFadeCoefficient()) - : 0.0f; -} - -const QVector& FaceTracker::getBlendshapeCoefficients() const { - static QVector blendshapes; - float fadeCoefficient = getFadeCoefficient(); - if (fadeCoefficient == 1.0f) { - return _blendshapeCoefficients; - } else { - blendshapes.resize(_blendshapeCoefficients.size()); - for (int i = 0; i < _blendshapeCoefficients.size(); i++) { - blendshapes[i] = glm::mix(0.0f, _blendshapeCoefficients[i], fadeCoefficient); - } - return blendshapes; - } -} - -float FaceTracker::getFadeCoefficient() const { - return _fadeCoefficient; -} - -const glm::vec3 FaceTracker::getHeadTranslation() const { - return glm::mix(glm::vec3(0.0f), _headTranslation, getFadeCoefficient()); -} - -const glm::quat FaceTracker::getHeadRotation() const { - return safeMix(glm::quat(), _headRotation, getFadeCoefficient()); -} - -void FaceTracker::update(float deltaTime) { - // Based on exponential distributions: http://en.wikipedia.org/wiki/Exponential_distribution - static const float EPSILON = 0.02f; // MUST BE < 1.0f - static const float INVERSE_AT_EPSILON = -std::log(EPSILON); // So that f(1.0f) = EPSILON ~ 0.0f - static const float RELAXATION_TIME = 0.8f; // sec - - if (isTracking()) { - if (_relaxationStatus == 1.0f) { - _fadeCoefficient = 1.0f; - return; - } - _relaxationStatus = glm::clamp(_relaxationStatus + deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = 1.0f - std::exp(-_relaxationStatus * INVERSE_AT_EPSILON); - } else { - if (_relaxationStatus == 0.0f) { - _fadeCoefficient = 0.0f; - return; - } - _relaxationStatus = glm::clamp(_relaxationStatus - deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = std::exp(-(1.0f - _relaxationStatus) * INVERSE_AT_EPSILON); - } -} - -void FaceTracker::reset() { - if (isActive() && !_isCalculatingFPS) { - QTimer::singleShot(FPS_TIMER_DELAY, this, SLOT(startFPSTimer())); - _isCalculatingFPS = true; - } -} - -void FaceTracker::startFPSTimer() { - _frameCount = 0; - QTimer::singleShot(FPS_TIMER_DURATION, this, SLOT(finishFPSTimer())); -} - -void FaceTracker::countFrame() { - if (_isCalculatingFPS) { - _frameCount++; - } -} - -void FaceTracker::finishFPSTimer() { - qCDebug(trackers) << "Face tracker FPS =" << (float)_frameCount / ((float)FPS_TIMER_DURATION / 1000.0f); - _isCalculatingFPS = false; -} - -void FaceTracker::toggleMute() { - _isMuted = !_isMuted; - emit muteToggled(); -} - -void FaceTracker::setEyeDeflection(float eyeDeflection) { - _eyeDeflection.set(eyeDeflection); -} - -void FaceTracker::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen, float mouth2, float mouth3, float mouth4, QVector& coefficients) { - const int MMMM_BLENDSHAPE = 34; - const int FUNNEL_BLENDSHAPE = 40; - const int SMILE_LEFT_BLENDSHAPE = 28; - const int SMILE_RIGHT_BLENDSHAPE = 29; - const int MAX_FAKE_BLENDSHAPE = 40; // Largest modified blendshape from above and below - - coefficients.resize(std::max((int)coefficients.size(), MAX_FAKE_BLENDSHAPE + 1)); - qFill(coefficients.begin(), coefficients.end(), 0.0f); - coefficients[_leftBlinkIndex] = leftBlink; - coefficients[_rightBlinkIndex] = rightBlink; - coefficients[_browUpCenterIndex] = browUp; - coefficients[_browUpLeftIndex] = browUp; - coefficients[_browUpRightIndex] = browUp; - coefficients[_jawOpenIndex] = jawOpen; - coefficients[SMILE_LEFT_BLENDSHAPE] = coefficients[SMILE_RIGHT_BLENDSHAPE] = mouth4; - coefficients[MMMM_BLENDSHAPE] = mouth2; - coefficients[FUNNEL_BLENDSHAPE] = mouth3; -} - diff --git a/libraries/trackers/src/trackers/FaceTracker.h b/libraries/trackers/src/trackers/FaceTracker.h deleted file mode 100644 index 47fbf72616..0000000000 --- a/libraries/trackers/src/trackers/FaceTracker.h +++ /dev/null @@ -1,131 +0,0 @@ -// -// Created by Andrzej Kapolka on 4/9/14. -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_FaceTracker_h -#define hifi_FaceTracker_h - -#include -#include - -#include -#include - -#include - -/// Base class for face trackers (DDE, BinaryVR). - -class FaceTracker : public QObject { - Q_OBJECT - -public: - virtual bool isActive() const { return false; } - virtual bool isTracking() const { return false; } - - virtual void init(); - virtual void update(float deltaTime); - virtual void reset(); - - float getFadeCoefficient() const; - - const glm::vec3 getHeadTranslation() const; - const glm::quat getHeadRotation() const; - - float getEstimatedEyePitch() const { return _estimatedEyePitch; } - float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } - - int getNumBlendshapes() const { return _blendshapeCoefficients.size(); } - bool isValidBlendshapeIndex(int index) const { return index >= 0 && index < getNumBlendshapes(); } - const QVector& getBlendshapeCoefficients() const; - float getBlendshapeCoefficient(int index) const; - - static bool isMuted() { return _isMuted; } - static void setIsMuted(bool isMuted) { _isMuted = isMuted; } - - static float getEyeDeflection() { return _eyeDeflection.get(); } - static void setEyeDeflection(float eyeDeflection); - - static void updateFakeCoefficients(float leftBlink, - float rightBlink, - float browUp, - float jawOpen, - float mouth2, - float mouth3, - float mouth4, - QVector& coefficients); - -signals: - - /**jsdoc - * @function FaceTracker.muteToggled - * @returns {Signal} - */ - void muteToggled(); - -public slots: - - // No JSDoc here because it's overridden in DdeFaceTracker. - virtual void setEnabled(bool enabled) = 0; - - /**jsdoc - * @function FaceTracker.toggleMute - */ - void toggleMute(); - - /**jsdoc - * @function FaceTracker.getMuted - * @returns {boolean} - */ - bool getMuted() { return _isMuted; } - -protected: - virtual ~FaceTracker() {}; - - bool _isInitialized = false; - static bool _isMuted; - - glm::vec3 _headTranslation = glm::vec3(0.0f); - glm::quat _headRotation = glm::quat(); - float _estimatedEyePitch = 0.0f; - float _estimatedEyeYaw = 0.0f; - QVector _blendshapeCoefficients; - - float _relaxationStatus = 0.0f; // Between 0.0f and 1.0f - float _fadeCoefficient = 0.0f; // Between 0.0f and 1.0f - - void countFrame(); - -private slots: - void startFPSTimer(); - void finishFPSTimer(); - -private: - bool _isCalculatingFPS = false; - int _frameCount = 0; - - // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes - static const int _leftBlinkIndex = 0; - static const int _rightBlinkIndex = 1; - static const int _leftEyeOpenIndex = 8; - static const int _rightEyeOpenIndex = 9; - - // Brows - static const int _browDownLeftIndex = 14; - static const int _browDownRightIndex = 15; - static const int _browUpCenterIndex = 16; - static const int _browUpLeftIndex = 17; - static const int _browUpRightIndex = 18; - - static const int _mouthSmileLeftIndex = 28; - static const int _mouthSmileRightIndex = 29; - - static const int _jawOpenIndex = 21; - - static Setting::Handle _eyeDeflection; -}; - -#endif // hifi_FaceTracker_h diff --git a/libraries/trackers/src/trackers/Logging.cpp b/libraries/trackers/src/trackers/Logging.cpp deleted file mode 100644 index a4dcf1b711..0000000000 --- a/libraries/trackers/src/trackers/Logging.cpp +++ /dev/null @@ -1,11 +0,0 @@ -// -// Created by Bradley Austin Davis on 2017/04/25 -// Copyright 2013-2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "Logging.h" - -Q_LOGGING_CATEGORY(trackers, "hifi.trackers") diff --git a/libraries/trackers/src/trackers/Logging.h b/libraries/trackers/src/trackers/Logging.h deleted file mode 100644 index 554429b61d..0000000000 --- a/libraries/trackers/src/trackers/Logging.h +++ /dev/null @@ -1,16 +0,0 @@ -// -// Created by Bradley Austin Davis on 2017/04/25 -// Copyright 2013-2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_TrackersLogging_h -#define hifi_TrackersLogging_h - -#include - -Q_DECLARE_LOGGING_CATEGORY(trackers) - -#endif // hifi_TrackersLogging_h diff --git a/tools/jsdoc/plugins/hifi.js b/tools/jsdoc/plugins/hifi.js index 07549530ce..8b18d02458 100644 --- a/tools/jsdoc/plugins/hifi.js +++ b/tools/jsdoc/plugins/hifi.js @@ -29,7 +29,6 @@ exports.handlers = { '../../interface/src/audio', '../../interface/src/avatar', '../../interface/src/commerce', - '../../interface/src/devices', '../../interface/src/java', '../../interface/src/networking', '../../interface/src/raypick', @@ -64,7 +63,6 @@ exports.handlers = { '../../libraries/shared/src', '../../libraries/shared/src/shared', '../../libraries/task/src/task', - '../../libraries/trackers/src/trackers', '../../libraries/ui/src', '../../libraries/ui/src/ui', '../../plugins/oculus/src', From 6efd74a339189433ea086807dfe05f41e3997a61 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 11 Oct 2019 09:37:01 -0700 Subject: [PATCH 07/33] WIP adding blendshapes to input system --- interface/resources/controllers/standard.json | 1 + .../controllers/standard_nomovement.json | 1 + interface/resources/controllers/vive.json | 2 + interface/src/avatar/MyHead.cpp | 10 ++- .../src/avatars-renderer/Head.cpp | 18 ++--- .../controllers/src/controllers/Actions.cpp | 68 ++++++++++++++++- .../controllers/src/controllers/Actions.h | 68 ++++++++++++++++- .../src/controllers/StandardController.cpp | 68 ++++++++++++++++- .../src/controllers/StandardControls.h | 3 + libraries/shared/src/BlendshapeConstants.cpp | 51 ++++++------- libraries/shared/src/BlendshapeConstants.h | 74 +++++++++++++++---- 11 files changed, 300 insertions(+), 64 deletions(-) diff --git a/interface/resources/controllers/standard.json b/interface/resources/controllers/standard.json index 195f909942..7f24924f7b 100644 --- a/interface/resources/controllers/standard.json +++ b/interface/resources/controllers/standard.json @@ -166,6 +166,7 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, + // AJT: blendshapes { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, diff --git a/interface/resources/controllers/standard_nomovement.json b/interface/resources/controllers/standard_nomovement.json index 602d3bb798..67eddfad31 100644 --- a/interface/resources/controllers/standard_nomovement.json +++ b/interface/resources/controllers/standard_nomovement.json @@ -61,6 +61,7 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, + // AJT: blendshapes { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, diff --git a/interface/resources/controllers/vive.json b/interface/resources/controllers/vive.json index b6fae1dd79..045f72739c 100644 --- a/interface/resources/controllers/vive.json +++ b/interface/resources/controllers/vive.json @@ -98,6 +98,8 @@ { "from": "Vive.Head", "to" : "Standard.Head" }, { "from": "Vive.LeftEye", "to" : "Standard.LeftEye" }, { "from": "Vive.RightEye", "to" : "Standard.RightEye" }, + + // AJT: blendshapes (only keep blink) { "from": "Vive.LeftEyeBlink", "to" : "Standard.LeftEyeBlink" }, { "from": "Vive.RightEyeBlink", "to" : "Standard.RightEyeBlink" }, diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 1c6e4690dd..73259fe073 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -58,6 +58,8 @@ void MyHead::simulate(float deltaTime) { // } // } + // AJT: blendshapes + auto userInputMapper = DependencyManager::get(); bool eyeLidsTracked = userInputMapper->getActionStateValid(controller::Action::LEFT_EYE_BLINK) && @@ -69,13 +71,13 @@ void MyHead::simulate(float deltaTime) { float leftEyeBlink = userInputMapper->getActionState(controller::Action::LEFT_EYE_BLINK); float rightEyeBlink = userInputMapper->getActionState(controller::Action::RIGHT_EYE_BLINK); _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[EYE_BLINK_INDICES[0]] = leftEyeBlink; - _blendshapeCoefficients[EYE_BLINK_INDICES[1]] = rightEyeBlink; + _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = leftEyeBlink; + _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = rightEyeBlink; } else { const float FULLY_OPEN = 0.0f; _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[EYE_BLINK_INDICES[0]] = FULLY_OPEN; - _blendshapeCoefficients[EYE_BLINK_INDICES[1]] = FULLY_OPEN; + _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = FULLY_OPEN; + _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = FULLY_OPEN; } } Parent::simulate(deltaTime); diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index ea6cdd7087..c7f807509c 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -262,26 +262,26 @@ void Head::applyEyelidOffset(glm::quat headOrientation) { float blinkUpCoefficient = -eyelidOffset; float blinkDownCoefficient = BLINK_DOWN_MULTIPLIER * eyelidOffset; - + float openUpCoefficient = eyelidOffset; float openDownCoefficient = OPEN_DOWN_MULTIPLIER * eyelidOffset; - + float browsUpCoefficient = BROW_UP_MULTIPLIER * eyelidOffset; float browsDownCoefficient = 0.0f; bool isLookingUp = (eyePitch > 0); - + if (isLookingUp) { for (int i = 0; i < 2; i++) { - _transientBlendshapeCoefficients[EYE_BLINK_INDICES[i]] = blinkUpCoefficient; - _transientBlendshapeCoefficients[EYE_OPEN_INDICES[i]] = openUpCoefficient; - _transientBlendshapeCoefficients[BROWS_U_INDICES[i]] = browsUpCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::EyeBlink_L + i] = blinkUpCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::EyeOpen_L + i] = openUpCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::BrowsU_L + i] = browsUpCoefficient; } } else { for (int i = 0; i < 2; i++) { - _transientBlendshapeCoefficients[EYE_BLINK_INDICES[i]] = blinkDownCoefficient; - _transientBlendshapeCoefficients[EYE_OPEN_INDICES[i]] = openDownCoefficient; - _transientBlendshapeCoefficients[BROWS_U_INDICES[i]] = browsDownCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::EyeBlink_L + i] = blinkDownCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::EyeOpen_L + i] = openDownCoefficient; + _transientBlendshapeCoefficients[(int)Blendshapes::BrowsU_L + i] = browsDownCoefficient; } } } diff --git a/libraries/controllers/src/controllers/Actions.cpp b/libraries/controllers/src/controllers/Actions.cpp index b6b96216b5..198c342b1d 100644 --- a/libraries/controllers/src/controllers/Actions.cpp +++ b/libraries/controllers/src/controllers/Actions.cpp @@ -349,8 +349,72 @@ namespace controller { makePosePair(Action::HEAD, "Head"), makePosePair(Action::LEFT_EYE, "LeftEye"), makePosePair(Action::RIGHT_EYE, "RightEye"), - makeAxisPair(Action::LEFT_EYE_BLINK, "LeftEyeBlink"), - makeAxisPair(Action::RIGHT_EYE_BLINK, "RightEyeBlink"), + + // AJT: blendshapes + makeAxisPair(Action::EYEBLINK_L, "EyeBlink_L"), + makeAxisPair(Action::EYEBLINK_R, "EyeBlink_R"), + makeAxisPair(Action::EYESQUINT_L, "EyeSquint_L"), + makeAxisPair(Action::EYESQUINT_R, "EyeSquint_R"), + makeAxisPair(Action::EYEDOWN_L, "EyeDown_L"), + makeAxisPair(Action::EYEDOWN_R, "EyeDown_R"), + makeAxisPair(Action::EYEIN_L, "EyeIn_L"), + makeAxisPair(Action::EYEIN_R, "EyeIn_R"), + makeAxisPair(Action::EYEOPEN_L, "EyeOpen_L"), + makeAxisPair(Action::EYEOPEN_R, "EyeOpen_R"), + makeAxisPair(Action::EYEOUT_L, "EyeOut_L"), + makeAxisPair(Action::EYEOUT_R, "EyeOut_R"), + makeAxisPair(Action::EYEUP_L, "EyeUp_L"), + makeAxisPair(Action::EYEUP_R, "EyeUp_R"), + makeAxisPair(Action::BROWSD_L, "BrowsD_L"), + makeAxisPair(Action::BROWSD_R, "BrowsD_R"), + makeAxisPair(Action::BROWSU_C, "BrowsU_C"), + makeAxisPair(Action::BROWSU_L, "BrowsU_L"), + makeAxisPair(Action::BROWSU_R, "BrowsU_R"), + makeAxisPair(Action::JAWFWD, "JawFwd"), + makeAxisPair(Action::JAWLEFT, "JawLeft"), + makeAxisPair(Action::JAWOPEN, "JawOpen"), + makeAxisPair(Action::JAWRIGHT, "JawRight"), + makeAxisPair(Action::MOUTHLEFT, "MouthLeft"), + makeAxisPair(Action::MOUTHRIGHT, "MouthRight"), + makeAxisPair(Action::MOUTHFROWN_L, "MouthFrown_L"), + makeAxisPair(Action::MOUTHFROWN_R, "MouthFrown_R"), + makeAxisPair(Action::MOUTHSMILE_L, "MouthSmile_L"), + makeAxisPair(Action::MOUTHSMILE_R, "MouthSmile_R"), + makeAxisPair(Action::MOUTHDIMPLE_L, "MouthDimple_L"), + makeAxisPair(Action::MOUTHDIMPLE_R, "MouthDimple_R"), + makeAxisPair(Action::LIPSSTRETCH_L, "LipsStretch_L"), + makeAxisPair(Action::LIPSSTRETCH_R, "LipsStretch_R"), + makeAxisPair(Action::LIPSUPPERCLOSE, "LipsUpperClose"), + makeAxisPair(Action::LIPSLOWERCLOSE, "LipsLowerClose"), + makeAxisPair(Action::LIPSUPPEROPEN, "LipsUpperOpen"), + makeAxisPair(Action::LIPSLOWEROPEN, "LipsLowerOpen"), + makeAxisPair(Action::LIPSFUNNEL, "LipsFunnel"), + makeAxisPair(Action::LIPSPUCKER, "LipsPucker"), + makeAxisPair(Action::PUFF, "Puff"), + makeAxisPair(Action::CHEEKSQUINT_L, "CheekSquint_L"), + makeAxisPair(Action::CHEEKSQUINT_R, "CheekSquint_R"), + makeAxisPair(Action::LIPSTOGETHER, "LipsTogether"), + makeAxisPair(Action::MOUTHUPPERUP_L, "MouthUpperUp_L"), + makeAxisPair(Action::MOUTHUPPERUP_R, "MouthUpperUp_R"), + makeAxisPair(Action::MOUTHLOWERDOWN_L, "MouthLowerDown_L"), + makeAxisPair(Action::MOUTHLOWERDOWN_R, "MouthLowerDown_R"), + makeAxisPair(Action::MOUTHPRESS_L, "MouthPress_L"), + makeAxisPair(Action::MOUTHPRESS_R, "MouthPress_R"), + makeAxisPair(Action::MOUTHSHRUGLOWER, "MouthShrugLower"), + makeAxisPair(Action::MOUTHSHRUGUPPER, "MouthShrugUpper"), + makeAxisPair(Action::NOSESNEER_L, "NoseSneer_L"), + makeAxisPair(Action::NOSESNEER_R, "NoseSneer_R"), + makeAxisPair(Action::TONGUEOUT, "TongueOut"), + makeAxisPair(Action::USERBLENDSHAPE0, "UserBlendshape0"), + makeAxisPair(Action::USERBLENDSHAPE1, "UserBlendshape1"), + makeAxisPair(Action::USERBLENDSHAPE2, "UserBlendshape2"), + makeAxisPair(Action::USERBLENDSHAPE3, "UserBlendshape3"), + makeAxisPair(Action::USERBLENDSHAPE4, "UserBlendshape4"), + makeAxisPair(Action::USERBLENDSHAPE5, "UserBlendshape5"), + makeAxisPair(Action::USERBLENDSHAPE6, "UserBlendshape6"), + makeAxisPair(Action::USERBLENDSHAPE7, "UserBlendshape7"), + makeAxisPair(Action::USERBLENDSHAPE8, "UserBlendshape8"), + makeAxisPair(Action::USERBLENDSHAPE9, "UserBlendshape9"), makePosePair(Action::LEFT_HAND_THUMB1, "LeftHandThumb1"), makePosePair(Action::LEFT_HAND_THUMB2, "LeftHandThumb2"), diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index f91d9f2522..e1ad17aafa 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -183,8 +183,72 @@ enum class Action { LEFT_EYE, RIGHT_EYE, - LEFT_EYE_BLINK, - RIGHT_EYE_BLINK, + + // AJT: blendshapes + EyeBlink_L, + EyeBlink_R, + EyeSquint_L, + EyeSquint_R, + EyeDown_L, + EyeDown_R, + EyeIn_L, + EyeIn_R, + EyeOpen_L, + EyeOpen_R, + EyeOut_L, + EyeOut_R, + EyeUp_L, + EyeUp_R, + BrowsD_L, + BrowsD_R, + BrowsU_C, + BrowsU_L, + BrowsU_R, + JawFwd, + JawLeft, + JawOpen, + JawRight, + MouthLeft, + MouthRight, + MouthFrown_L, + MouthFrown_R, + MouthSmile_L, + MouthSmile_R, + MouthDimple_L, + MouthDimple_R, + LipsStretch_L, + LipsStretch_R, + LipsUpperClose, + LipsLowerClose, + LipsUpperOpen, + LipsLowerOpen, + LipsFunnel, + LipsPucker, + Puff, + CheekSquint_L, + CheekSquint_R, + LipsTogether, + MouthUpperUp_L, + MouthUpperUp_R, + MouthLowerDown_L, + MouthLowerDown_R, + MouthPress_L, + MouthPress_R, + MouthShrugLower, + MouthShrugUpper, + NoseSneer_L, + NoseSneer_R, + TongueOut, + UserBlendshape0, + UserBlendshape1, + UserBlendshape2, + UserBlendshape3, + UserBlendshape4, + UserBlendshape5, + UserBlendshape6, + UserBlendshape7, + UserBlendshape8, + UserBlendshape9, NUM_ACTIONS }; diff --git a/libraries/controllers/src/controllers/StandardController.cpp b/libraries/controllers/src/controllers/StandardController.cpp index ae592485dc..14697b26d2 100644 --- a/libraries/controllers/src/controllers/StandardController.cpp +++ b/libraries/controllers/src/controllers/StandardController.cpp @@ -355,8 +355,72 @@ Input::NamedVector StandardController::getAvailableInputs() const { makePair(HEAD, "Head"), makePair(LEFT_EYE, "LeftEye"), makePair(RIGHT_EYE, "RightEye"), - makePair(LEFT_EYE_BLINK, "LeftEyeBlink"), - makePair(RIGHT_EYE_BLINK, "RightEyeBlink"), + + // AJT: blendshapes + makePair(EYEBLINK_L, "EyeBlink_L"), + makePair(EYEBLINK_R, "EyeBlink_R"), + makePair(EYESQUINT_L, "EyeSquint_L"), + makePair(EYESQUINT_R, "EyeSquint_R"), + makePair(EYEDOWN_L, "EyeDown_L"), + makePair(EYEDOWN_R, "EyeDown_R"), + makePair(EYEIN_L, "EyeIn_L"), + makePair(EYEIN_R, "EyeIn_R"), + makePair(EYEOPEN_L, "EyeOpen_L"), + makePair(EYEOPEN_R, "EyeOpen_R"), + makePair(EYEOUT_L, "EyeOut_L"), + makePair(EYEOUT_R, "EyeOut_R"), + makePair(EYEUP_L, "EyeUp_L"), + makePair(EYEUP_R, "EyeUp_R"), + makePair(BROWSD_L, "BrowsD_L"), + makePair(BROWSD_R, "BrowsD_R"), + makePair(BROWSU_C, "BrowsU_C"), + makePair(BROWSU_L, "BrowsU_L"), + makePair(BROWSU_R, "BrowsU_R"), + makePair(JAWFWD, "JawFwd"), + makePair(JAWLEFT, "JawLeft"), + makePair(JAWOPEN, "JawOpen"), + makePair(JAWRIGHT, "JawRight"), + makePair(MOUTHLEFT, "MouthLeft"), + makePair(MOUTHRIGHT, "MouthRight"), + makePair(MOUTHFROWN_L, "MouthFrown_L"), + makePair(MOUTHFROWN_R, "MouthFrown_R"), + makePair(MOUTHSMILE_L, "MouthSmile_L"), + makePair(MOUTHSMILE_R, "MouthSmile_R"), + makePair(MOUTHDIMPLE_L, "MouthDimple_L"), + makePair(MOUTHDIMPLE_R, "MouthDimple_R"), + makePair(LIPSSTRETCH_L, "LipsStretch_L"), + makePair(LIPSSTRETCH_R, "LipsStretch_R"), + makePair(LIPSUPPERCLOSE, "LipsUpperClose"), + makePair(LIPSLOWERCLOSE, "LipsLowerClose"), + makePair(LIPSUPPEROPEN, "LipsUpperOpen"), + makePair(LIPSLOWEROPEN, "LipsLowerOpen"), + makePair(LIPSFUNNEL, "LipsFunnel"), + makePair(LIPSPUCKER, "LipsPucker"), + makePair(PUFF, "Puff"), + makePair(CHEEKSQUINT_L, "CheekSquint_L"), + makePair(CHEEKSQUINT_R, "CheekSquint_R"), + makePair(LIPSTOGETHER, "LipsTogether"), + makePair(MOUTHUPPERUP_L, "MouthUpperUp_L"), + makePair(MOUTHUPPERUP_R, "MouthUpperUp_R"), + makePair(MOUTHLOWERDOWN_L, "MouthLowerDown_L"), + makePair(MOUTHLOWERDOWN_R, "MouthLowerDown_R"), + makePair(MOUTHPRESS_L, "MouthPress_L"), + makePair(MOUTHPRESS_R, "MouthPress_R"), + makePair(MOUTHSHRUGLOWER, "MouthShrugLower"), + makePair(MOUTHSHRUGUPPER, "MouthShrugUpper"), + makePair(NOSESNEER_L, "NoseSneer_L"), + makePair(NOSESNEER_R, "NoseSneer_R"), + makePair(TONGUEOUT, "TongueOut"), + makePair(USERBLENDSHAPE0, "UserBlendshape0"), + makePair(USERBLENDSHAPE1, "UserBlendshape1"), + makePair(USERBLENDSHAPE2, "UserBlendshape2"), + makePair(USERBLENDSHAPE3, "UserBlendshape3"), + makePair(USERBLENDSHAPE4, "UserBlendshape4"), + makePair(USERBLENDSHAPE5, "UserBlendshape5"), + makePair(USERBLENDSHAPE6, "UserBlendshape6"), + makePair(USERBLENDSHAPE7, "UserBlendshape7"), + makePair(USERBLENDSHAPE8, "UserBlendshape8"), + makePair(USERBLENDSHAPE9, "UserBlendshape9"), // Aliases, PlayStation style names makePair(LB, "L1"), diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index 99d9246264..2af822ffc2 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -90,8 +90,11 @@ namespace controller { // Grips LEFT_GRIP, RIGHT_GRIP, + + // AJT: blendshapes LEFT_EYE_BLINK, RIGHT_EYE_BLINK, + NUM_STANDARD_AXES, LZ = LT, RZ = RT diff --git a/libraries/shared/src/BlendshapeConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp index 528b941b73..7564c31944 100644 --- a/libraries/shared/src/BlendshapeConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -34,7 +34,6 @@ const char* FACESHIFT_BLENDSHAPES[] = { "JawFwd", "JawLeft", "JawOpen", - "JawChew", "JawRight", "MouthLeft", "MouthRight", @@ -48,40 +47,34 @@ const char* FACESHIFT_BLENDSHAPES[] = { "LipsStretch_R", "LipsUpperClose", "LipsLowerClose", - "LipsUpperUp", - "LipsLowerDown", "LipsUpperOpen", "LipsLowerOpen", "LipsFunnel", "LipsPucker", - "ChinLowerRaise", - "ChinUpperRaise", - "Sneer", "Puff", "CheekSquint_L", "CheekSquint_R", + "LipsTogether", + "MouthUpperUp_L", + "MouthUpperUp_R", + "MouthLowerDown_L", + "MouthLowerDown_R", + "MouthPress_L", + "MouthPress_R", + "MouthShrugLower", + "MouthShrugUpper", + "NoseSneer_L", + "NoseSneer_R", + "TongueOut", + "UserBlendshape0", + "UserBlendshape1", + "UserBlendshape2", + "UserBlendshape3", + "UserBlendshape4", + "UserBlendshape5", + "UserBlendshape6", + "UserBlendshape7", + "UserBlendshape8", + "UserBlendshape9", "" }; - -// new in ARKit -// LipsTogether -// MouthPressLeft -// MouthPressRight -// MouthShrugLower -// MouthShrugUpper -// TongueOut - -const int EYE_BLINK_L_INDEX = 0; -const int EYE_BLINK_R_INDEX = 1; -const int EYE_SQUINT_L_INDEX = 2; -const int EYE_SQUINT_R_INDEX = 3; -const int EYE_OPEN_L_INDEX = 8; -const int EYE_OPEN_R_INDEX = 9; -const int BROWS_U_L_INDEX = 17; -const int BROWS_U_R_INDEX = 18; - - -const int EYE_BLINK_INDICES[] = { EYE_BLINK_L_INDEX, EYE_BLINK_R_INDEX }; -const int EYE_SQUINT_INDICES[] = { EYE_SQUINT_L_INDEX, EYE_SQUINT_R_INDEX }; -const int EYE_OPEN_INDICES[] = { EYE_OPEN_L_INDEX, EYE_OPEN_R_INDEX }; -const int BROWS_U_INDICES[] = { BROWS_U_L_INDEX, BROWS_U_R_INDEX }; diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 0fca03e34b..97b459fa0e 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -15,12 +15,6 @@ /// The names of the blendshapes expected by Faceshift, terminated with an empty string. extern const char* FACESHIFT_BLENDSHAPES[]; -// Eyes and Brows indices -extern const int EYE_BLINK_INDICES[]; -extern const int EYE_OPEN_INDICES[]; -extern const int BROWS_U_INDICES[]; -extern const int EYE_SQUINT_INDICES[]; - enum class Blendshapes : int { EyeBlink_L = 0, EyeBlink_R, @@ -32,7 +26,7 @@ enum class Blendshapes : int { EyeIn_R, EyeOpen_L, EyeOpen_R, - EyeOut_L, // 10 + EyeOut_L, EyeOut_R, EyeUp_L, EyeUp_R, @@ -42,9 +36,8 @@ enum class Blendshapes : int { BrowsU_L, BrowsU_R, JawFwd, - JawLeft, // 20 + JawLeft, JawOpen, - JawChew, // legacy not in ARKit JawRight, MouthLeft, MouthRight, @@ -52,25 +45,74 @@ enum class Blendshapes : int { MouthFrown_R, MouthSmile_L, MouthSmile_R, - MouthDimple_L, // 30 + MouthDimple_L, MouthDimple_R, LipsStretch_L, LipsStretch_R, LipsUpperClose, LipsLowerClose, - LipsUpperUp, // legacy, split in ARKit - LipsLowerDown, // legacy, split in ARKit LipsUpperOpen, LipsLowerOpen, - LipsFunnel, // 40 + LipsFunnel, LipsPucker, - ChinLowerRaise, - ChinUpperRaise, - Sneer, // legacy, split in ARKit Puff, CheekSquint_L, CheekSquint_R, + LipsTogether, + MouthUpperUp_L, + MouthUpperUp_R, + MouthLowerDown_L, + MouthLowerDown_R, + MouthPress_L, + MouthPress_R, + MouthShrugLower, + MouthShrugUpper, + NoseSneer_L, + NoseSneer_R, + TongueOut, + UserBlendshape0, + UserBlendshape1, + UserBlendshape2, + UserBlendshape3, + UserBlendshape4, + UserBlendshape5, + UserBlendshape6, + UserBlendshape7, + UserBlendshape8, + UserBlendshape9, BlendshapeCount }; +enum class LegacyBlendshpaes : int { + JawChew, // not in ARKit + LipsUpperUp, // split in ARKit + LipsLowerDown, // split in ARKit + ChinLowerRaise, // not in ARKit + ChinUpperRaise, // not in ARKit + Sneer, // split in ARKit + LegacyBlendshapeCount +} + +// NEW in ARKit +// * LipsTogether +// * MouthUpperUp_L +// * MouthUpperUp_R +// * MouthLowerDown_L +// * MouthLowerDown_R +// * MouthPress_L +// * MouthPress_R +// * MouthShrugLower +// * MouthShrugUpper +// * NoseSneer_L +// * NoseSneer_R +// * TongueOut + +// Legacy shapes +// * JawChew (not in ARKit) +// * MouthUpperUp (split in ARKit) +// * MouthLowerDown (split in ARKit) +// * Sneer (split in ARKit) +// * ChinLowerRaise (not in ARKit) +// * ChinUpperRase (not in ARKit) + #endif // hifi_BlendshapeConstants_h From 389f5a1d335ad4c88f358fb8ff7356a74741cac6 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 15 Oct 2019 11:37:10 -0700 Subject: [PATCH 08/33] ARKit Blendshape hookup work --- interface/resources/controllers/standard.json | 68 ++++++++- .../controllers/standard_nomovement.json | 67 ++++++++- interface/resources/controllers/vive.json | 5 +- interface/src/avatar/MyAvatar.cpp | 12 -- interface/src/avatar/MyHead.cpp | 142 ++++++++++++++---- interface/src/avatar/MySkeletonModel.cpp | 5 + .../src/avatars-renderer/Head.cpp | 12 +- .../src/avatars-renderer/SkeletonModel.cpp | 4 + libraries/avatars/src/AvatarData.cpp | 79 ++++++---- libraries/avatars/src/AvatarData.h | 11 +- libraries/avatars/src/HeadData.cpp | 60 ++++---- libraries/avatars/src/HeadData.h | 43 ++++-- .../controllers/src/controllers/Actions.h | 128 ++++++++-------- .../src/controllers/StandardControls.h | 66 +++++++- .../networking/src/udt/PacketHeaders.cpp | 4 +- libraries/networking/src/udt/PacketHeaders.h | 3 +- libraries/shared/src/BlendshapeConstants.h | 2 +- 17 files changed, 507 insertions(+), 204 deletions(-) diff --git a/interface/resources/controllers/standard.json b/interface/resources/controllers/standard.json index 7f24924f7b..8c84039b86 100644 --- a/interface/resources/controllers/standard.json +++ b/interface/resources/controllers/standard.json @@ -166,10 +166,70 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, - // AJT: blendshapes - { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, - { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, - + { "from": "Standard.EyeBlink_L", "to": "Actions.EyeBlink_L" }, + { "from": "Standard.EyeBlink_R", "to": "Actions.EyeBlink_R" }, + { "from": "Standard.EyeSquint_L", "to": "Actions.EyeSquint_L" }, + { "from": "Standard.EyeSquint_R", "to": "Actions.EyeSquint_R" }, + { "from": "Standard.EyeDown_L", "to": "Actions.EyeDown_L" }, + { "from": "Standard.EyeDown_R", "to": "Actions.EyeDown_R" }, + { "from": "Standard.EyeIn_L", "to": "Actions.EyeIn_L" }, + { "from": "Standard.EyeIn_R", "to": "Actions.EyeIn_R" }, + { "from": "Standard.EyeOpen_L", "to": "Actions.EyeOpen_L" }, + { "from": "Standard.EyeOpen_R", "to": "Actions.EyeOpen_R" }, + { "from": "Standard.EyeOut_L", "to": "Actions.EyeOut_L" }, + { "from": "Standard.EyeOut_R", "to": "Actions.EyeOut_R" }, + { "from": "Standard.EyeUp_L", "to": "Actions.EyeUp_L" }, + { "from": "Standard.EyeUp_R", "to": "Actions.EyeUp_R" }, + { "from": "Standard.BrowsD_L", "to": "Actions.BrowsD_L" }, + { "from": "Standard.BrowsD_R", "to": "Actions.BrowsD_R" }, + { "from": "Standard.BrowsU_C", "to": "Actions.BrowsU_C" }, + { "from": "Standard.BrowsU_L", "to": "Actions.BrowsU_L" }, + { "from": "Standard.BrowsU_R", "to": "Actions.BrowsU_R" }, + { "from": "Standard.JawFwd", "to": "Actions.JawFwd" }, + { "from": "Standard.JawLeft", "to": "Actions.JawLeft" }, + { "from": "Standard.JawOpen", "to": "Actions.JawOpen" }, + { "from": "Standard.JawRight", "to": "Actions.JawRight" }, + { "from": "Standard.MouthLeft", "to": "Actions.MouthLeft" }, + { "from": "Standard.MouthRight", "to": "Actions.MouthRight" }, + { "from": "Standard.MouthFrown_L", "to": "Actions.MouthFrown_L" }, + { "from": "Standard.MouthFrown_R", "to": "Actions.MouthFrown_R" }, + { "from": "Standard.MouthSmile_L", "to": "Actions.MouthSmile_L" }, + { "from": "Standard.MouthSmile_R", "to": "Actions.MouthSmile_R" }, + { "from": "Standard.MouthDimple_L", "to": "Actions.MouthDimple_L" }, + { "from": "Standard.MouthDimple_R", "to": "Actions.MouthDimple_R" }, + { "from": "Standard.LipsStretch_L", "to": "Actions.LipsStretch_L" }, + { "from": "Standard.LipsStretch_R", "to": "Actions.LipsStretch_R" }, + { "from": "Standard.LipsUpperClose", "to": "Actions.LipsUpperClose" }, + { "from": "Standard.LipsLowerClose", "to": "Actions.LipsLowerClose" }, + { "from": "Standard.LipsUpperOpen", "to": "Actions.LipsUpperOpen" }, + { "from": "Standard.LipsLowerOpen", "to": "Actions.LipsLowerOpen" }, + { "from": "Standard.LipsFunnel", "to": "Actions.LipsFunnel" }, + { "from": "Standard.LipsPucker", "to": "Actions.LipsPucker" }, + { "from": "Standard.Puff", "to": "Actions.Puff" }, + { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, + { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, + { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, + { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, + { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, + { "from": "Standard.MouthLowerDown_R", "to": "Actions.MouthLowerDown_R" }, + { "from": "Standard.MouthPress_L", "to": "Actions.MouthPress_L" }, + { "from": "Standard.MouthPress_R", "to": "Actions.MouthPress_R" }, + { "from": "Standard.MouthShrugLower", "to": "Actions.MouthShrugLower" }, + { "from": "Standard.MouthShrugUpper", "to": "Actions.MouthShrugUpper" }, + { "from": "Standard.NoseSneer_L", "to": "Actions.NoseSneer_L" }, + { "from": "Standard.NoseSneer_R", "to": "Actions.NoseSneer_R" }, + { "from": "Standard.TongueOut", "to": "Actions.TongueOut" }, + { "from": "Standard.UserBlendshape0", "to": "Actions.UserBlendshape0" }, + { "from": "Standard.UserBlendshape1", "to": "Actions.UserBlendshape1" }, + { "from": "Standard.UserBlendshape2", "to": "Actions.UserBlendshape2" }, + { "from": "Standard.UserBlendshape3", "to": "Actions.UserBlendshape3" }, + { "from": "Standard.UserBlendshape4", "to": "Actions.UserBlendshape4" }, + { "from": "Standard.UserBlendshape5", "to": "Actions.UserBlendshape5" }, + { "from": "Standard.UserBlendshape6", "to": "Actions.UserBlendshape6" }, + { "from": "Standard.UserBlendshape7", "to": "Actions.UserBlendshape7" }, + { "from": "Standard.UserBlendshape8", "to": "Actions.UserBlendshape8" }, + { "from": "Standard.UserBlendshape9", "to": "Actions.UserBlendshape9" }, { "from": "Standard.TrackedObject00", "to" : "Actions.TrackedObject00" }, { "from": "Standard.TrackedObject01", "to" : "Actions.TrackedObject01" }, diff --git a/interface/resources/controllers/standard_nomovement.json b/interface/resources/controllers/standard_nomovement.json index 67eddfad31..e311a5fa62 100644 --- a/interface/resources/controllers/standard_nomovement.json +++ b/interface/resources/controllers/standard_nomovement.json @@ -61,9 +61,70 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, - // AJT: blendshapes - { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, - { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, + { "from": "Standard.EyeBlink_L", "to": "Actions.EyeBlink_L" }, + { "from": "Standard.EyeBlink_R", "to": "Actions.EyeBlink_R" }, + { "from": "Standard.EyeSquint_L", "to": "Actions.EyeSquint_L" }, + { "from": "Standard.EyeSquint_R", "to": "Actions.EyeSquint_R" }, + { "from": "Standard.EyeDown_L", "to": "Actions.EyeDown_L" }, + { "from": "Standard.EyeDown_R", "to": "Actions.EyeDown_R" }, + { "from": "Standard.EyeIn_L", "to": "Actions.EyeIn_L" }, + { "from": "Standard.EyeIn_R", "to": "Actions.EyeIn_R" }, + { "from": "Standard.EyeOpen_L", "to": "Actions.EyeOpen_L" }, + { "from": "Standard.EyeOpen_R", "to": "Actions.EyeOpen_R" }, + { "from": "Standard.EyeOut_L", "to": "Actions.EyeOut_L" }, + { "from": "Standard.EyeOut_R", "to": "Actions.EyeOut_R" }, + { "from": "Standard.EyeUp_L", "to": "Actions.EyeUp_L" }, + { "from": "Standard.EyeUp_R", "to": "Actions.EyeUp_R" }, + { "from": "Standard.BrowsD_L", "to": "Actions.BrowsD_L" }, + { "from": "Standard.BrowsD_R", "to": "Actions.BrowsD_R" }, + { "from": "Standard.BrowsU_C", "to": "Actions.BrowsU_C" }, + { "from": "Standard.BrowsU_L", "to": "Actions.BrowsU_L" }, + { "from": "Standard.BrowsU_R", "to": "Actions.BrowsU_R" }, + { "from": "Standard.JawFwd", "to": "Actions.JawFwd" }, + { "from": "Standard.JawLeft", "to": "Actions.JawLeft" }, + { "from": "Standard.JawOpen", "to": "Actions.JawOpen" }, + { "from": "Standard.JawRight", "to": "Actions.JawRight" }, + { "from": "Standard.MouthLeft", "to": "Actions.MouthLeft" }, + { "from": "Standard.MouthRight", "to": "Actions.MouthRight" }, + { "from": "Standard.MouthFrown_L", "to": "Actions.MouthFrown_L" }, + { "from": "Standard.MouthFrown_R", "to": "Actions.MouthFrown_R" }, + { "from": "Standard.MouthSmile_L", "to": "Actions.MouthSmile_L" }, + { "from": "Standard.MouthSmile_R", "to": "Actions.MouthSmile_R" }, + { "from": "Standard.MouthDimple_L", "to": "Actions.MouthDimple_L" }, + { "from": "Standard.MouthDimple_R", "to": "Actions.MouthDimple_R" }, + { "from": "Standard.LipsStretch_L", "to": "Actions.LipsStretch_L" }, + { "from": "Standard.LipsStretch_R", "to": "Actions.LipsStretch_R" }, + { "from": "Standard.LipsUpperClose", "to": "Actions.LipsUpperClose" }, + { "from": "Standard.LipsLowerClose", "to": "Actions.LipsLowerClose" }, + { "from": "Standard.LipsUpperOpen", "to": "Actions.LipsUpperOpen" }, + { "from": "Standard.LipsLowerOpen", "to": "Actions.LipsLowerOpen" }, + { "from": "Standard.LipsFunnel", "to": "Actions.LipsFunnel" }, + { "from": "Standard.LipsPucker", "to": "Actions.LipsPucker" }, + { "from": "Standard.Puff", "to": "Actions.Puff" }, + { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, + { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, + { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, + { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, + { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, + { "from": "Standard.MouthLowerDown_R", "to": "Actions.MouthLowerDown_R" }, + { "from": "Standard.MouthPress_L", "to": "Actions.MouthPress_L" }, + { "from": "Standard.MouthPress_R", "to": "Actions.MouthPress_R" }, + { "from": "Standard.MouthShrugLower", "to": "Actions.MouthShrugLower" }, + { "from": "Standard.MouthShrugUpper", "to": "Actions.MouthShrugUpper" }, + { "from": "Standard.NoseSneer_L", "to": "Actions.NoseSneer_L" }, + { "from": "Standard.NoseSneer_R", "to": "Actions.NoseSneer_R" }, + { "from": "Standard.TongueOut", "to": "Actions.TongueOut" }, + { "from": "Standard.UserBlendshape0", "to": "Actions.UserBlendshape0" }, + { "from": "Standard.UserBlendshape1", "to": "Actions.UserBlendshape1" }, + { "from": "Standard.UserBlendshape2", "to": "Actions.UserBlendshape2" }, + { "from": "Standard.UserBlendshape3", "to": "Actions.UserBlendshape3" }, + { "from": "Standard.UserBlendshape4", "to": "Actions.UserBlendshape4" }, + { "from": "Standard.UserBlendshape5", "to": "Actions.UserBlendshape5" }, + { "from": "Standard.UserBlendshape6", "to": "Actions.UserBlendshape6" }, + { "from": "Standard.UserBlendshape7", "to": "Actions.UserBlendshape7" }, + { "from": "Standard.UserBlendshape8", "to": "Actions.UserBlendshape8" }, + { "from": "Standard.UserBlendshape9", "to": "Actions.UserBlendshape9" }, { "from": "Standard.TrackedObject00", "to" : "Actions.TrackedObject00" }, { "from": "Standard.TrackedObject01", "to" : "Actions.TrackedObject01" }, diff --git a/interface/resources/controllers/vive.json b/interface/resources/controllers/vive.json index 045f72739c..4090256418 100644 --- a/interface/resources/controllers/vive.json +++ b/interface/resources/controllers/vive.json @@ -99,9 +99,8 @@ { "from": "Vive.LeftEye", "to" : "Standard.LeftEye" }, { "from": "Vive.RightEye", "to" : "Standard.RightEye" }, - // AJT: blendshapes (only keep blink) - { "from": "Vive.LeftEyeBlink", "to" : "Standard.LeftEyeBlink" }, - { "from": "Vive.RightEyeBlink", "to" : "Standard.RightEyeBlink" }, + { "from": "Vive.EyeBlink_L", "to" : "Standard.EyeBlink_L" }, + { "from": "Vive.EyeBlink_R", "to" : "Standard.EyeBlink_R" }, { "from": "Vive.LeftFoot", "to" : "Standard.LeftFoot", diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index fb42f89048..d697e1869a 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -780,18 +780,6 @@ void MyAvatar::update(float deltaTime) { emit energyChanged(currentEnergy); updateEyeContactTarget(deltaTime); - - // if we're getting eye rotations from a tracker, disable observer-side procedural eye motions - auto userInputMapper = DependencyManager::get(); - bool eyesTracked = - userInputMapper->getPoseState(controller::Action::LEFT_EYE).valid && - userInputMapper->getPoseState(controller::Action::RIGHT_EYE).valid; - - int leftEyeJointIndex = getJointIndex("LeftEye"); - int rightEyeJointIndex = getJointIndex("RightEye"); - bool eyesAreOverridden = getIsJointOverridden(leftEyeJointIndex) || getIsJointOverridden(rightEyeJointIndex); - - _headData->setHasProceduralEyeMovement(!(eyesTracked || eyesAreOverridden)); } void MyAvatar::updateEyeContactTarget(float deltaTime) { diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 73259fe073..f92f8b7218 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -21,6 +21,73 @@ using namespace std; +static controller::Action blendshapeActions[] = { + controller::Action::EYEBLINK_L, + controller::Action::EYEBLINK_R, + controller::Action::EYESQUINT_L, + controller::Action::EYESQUINT_R, + controller::Action::EYEDOWN_L, + controller::Action::EYEDOWN_R, + controller::Action::EYEIN_L, + controller::Action::EYEIN_R, + controller::Action::EYEOPEN_L, + controller::Action::EYEOPEN_R, + controller::Action::EYEOUT_L, + controller::Action::EYEOUT_R, + controller::Action::EYEUP_L, + controller::Action::EYEUP_R, + controller::Action::BROWSD_L, + controller::Action::BROWSD_R, + controller::Action::BROWSU_C, + controller::Action::BROWSU_L, + controller::Action::BROWSU_R, + controller::Action::JAWFWD, + controller::Action::JAWLEFT, + controller::Action::JAWOPEN, + controller::Action::JAWRIGHT, + controller::Action::MOUTHLEFT, + controller::Action::MOUTHRIGHT, + controller::Action::MOUTHFROWN_L, + controller::Action::MOUTHFROWN_R, + controller::Action::MOUTHSMILE_L, + controller::Action::MOUTHSMILE_R, + controller::Action::MOUTHDIMPLE_L, + controller::Action::MOUTHDIMPLE_R, + controller::Action::LIPSSTRETCH_L, + controller::Action::LIPSSTRETCH_R, + controller::Action::LIPSUPPERCLOSE, + controller::Action::LIPSLOWERCLOSE, + controller::Action::LIPSUPPEROPEN, + controller::Action::LIPSLOWEROPEN, + controller::Action::LIPSFUNNEL, + controller::Action::LIPSPUCKER, + controller::Action::PUFF, + controller::Action::CHEEKSQUINT_L, + controller::Action::CHEEKSQUINT_R, + controller::Action::LIPSTOGETHER, + controller::Action::MOUTHUPPERUP_L, + controller::Action::MOUTHUPPERUP_R, + controller::Action::MOUTHLOWERDOWN_L, + controller::Action::MOUTHLOWERDOWN_R, + controller::Action::MOUTHPRESS_L, + controller::Action::MOUTHPRESS_R, + controller::Action::MOUTHSHRUGLOWER, + controller::Action::MOUTHSHRUGUPPER, + controller::Action::NOSESNEER_L, + controller::Action::NOSESNEER_R, + controller::Action::TONGUEOUT, + controller::Action::USERBLENDSHAPE0, + controller::Action::USERBLENDSHAPE1, + controller::Action::USERBLENDSHAPE2, + controller::Action::USERBLENDSHAPE3, + controller::Action::USERBLENDSHAPE4, + controller::Action::USERBLENDSHAPE5, + controller::Action::USERBLENDSHAPE6, + controller::Action::USERBLENDSHAPE7, + controller::Action::USERBLENDSHAPE8, + controller::Action::USERBLENDSHAPE9 +}; + MyHead::MyHead(MyAvatar* owningAvatar) : Head(owningAvatar) { } @@ -44,40 +111,57 @@ void MyHead::simulate(float deltaTime) { auto player = DependencyManager::get(); // Only use face trackers when not playing back a recording. if (!player->isPlaying()) { - // TODO -- finish removing face-tracker specific code. To do this, add input channels for - // each blendshape-coefficient and update the various json files to relay them in a useful way. - // After that, input plugins can be used to drive the avatar's face, and the various "DDE" files - // can be ported into the plugin and removed. - // - // auto faceTracker = qApp->getActiveFaceTracker(); - // const bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); - // _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); - // if (_isFaceTrackerConnected) { - // if (hasActualFaceTrackerConnected) { - // _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); - // } - // } - - // AJT: blendshapes auto userInputMapper = DependencyManager::get(); + + // if input system has control over blink blendshapes bool eyeLidsTracked = - userInputMapper->getActionStateValid(controller::Action::LEFT_EYE_BLINK) && - userInputMapper->getActionStateValid(controller::Action::RIGHT_EYE_BLINK); + userInputMapper->getActionStateValid(controller::Action::EYEBLINK_L) || + userInputMapper->getActionStateValid(controller::Action::EYEBLINK_R); - setHasScriptedBlendshapes(eyeLidsTracked); + // if input system has control over the brows. + bool browsTracked = + userInputMapper->getActionStateValid(controller::Action::BROWSD_L) || + userInputMapper->getActionStateValid(controller::Action::BROWSD_R) || + userInputMapper->getActionStateValid(controller::Action::BROWSU_L) || + userInputMapper->getActionStateValid(controller::Action::BROWSU_R) || + userInputMapper->getActionStateValid(controller::Action::BROWSU_C); - if (eyeLidsTracked) { - float leftEyeBlink = userInputMapper->getActionState(controller::Action::LEFT_EYE_BLINK); - float rightEyeBlink = userInputMapper->getActionState(controller::Action::RIGHT_EYE_BLINK); - _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = leftEyeBlink; - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = rightEyeBlink; - } else { - const float FULLY_OPEN = 0.0f; - _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = FULLY_OPEN; - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = FULLY_OPEN; + // if input system has control of mouth + bool mouthTracked = + userInputMapper->getActionStateValid(controller::Action::JAWOPEN) || + userInputMapper->getActionStateValid(controller::Action::LIPSUPPERCLOSE) || + userInputMapper->getActionStateValid(controller::Action::LIPSLOWERCLOSE) || + userInputMapper->getActionStateValid(controller::Action::LIPSFUNNEL) || + userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_L) || + userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_R); + + bool eyesTracked = + userInputMapper->getPoseState(controller::Action::LEFT_EYE).valid && + userInputMapper->getPoseState(controller::Action::RIGHT_EYE).valid; + + MyAvatar* myAvatar = static_cast(_owningAvatar); + int leftEyeJointIndex = myAvatar->getJointIndex("LeftEye"); + int rightEyeJointIndex = myAvatar->getJointIndex("RightEye"); + bool eyeJointsOverridden = myAvatar->getIsJointOverridden(leftEyeJointIndex) || myAvatar->getIsJointOverridden(rightEyeJointIndex); + + bool anyInputTracked = false; + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { + anyInputTracked = anyInputTracked || userInputMapper->getActionStateValid(blendshapeActions[i]); + } + + setHasInputDrivenBlendshapes(anyInputTracked); + + // suppress any procedural blendshape animation if they overlap with driven input. + setSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, eyeLidsTracked); + setSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, eyeLidsTracked || browsTracked); + setSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, mouthTracked); + setSuppressProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation, eyesTracked || eyeJointsOverridden); + + if (anyInputTracked) { + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { + _blendshapeCoefficients[i] = userInputMapper->getActionState(blendshapeActions[i]); + } } } Parent::simulate(deltaTime); diff --git a/interface/src/avatar/MySkeletonModel.cpp b/interface/src/avatar/MySkeletonModel.cpp index 8d92767321..0d7bd3f3b3 100755 --- a/interface/src/avatar/MySkeletonModel.cpp +++ b/interface/src/avatar/MySkeletonModel.cpp @@ -114,6 +114,9 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { Head* head = _owningAvatar->getHead(); + // AJT: blendshapes TODO: RE-enable this and avoid duplication with + // SkeletonModel::updateRig() + /* bool eyePosesValid = !head->getHasProceduralEyeMovement(); glm::vec3 lookAt; if (eyePosesValid) { @@ -121,6 +124,8 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { } else { lookAt = avoidCrossedEyes(head->getLookAtPosition()); } + */ + glm::vec3 lookAt = avoidCrossedEyes(head->getLookAtPosition()); MyAvatar* myAvatar = static_cast(_owningAvatar); assert(myAvatar); diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index c7f807509c..2530fea4e7 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -72,7 +72,8 @@ void Head::simulate(float deltaTime) { _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); } - if (getHasProceduralEyeMovement()) { + if (getProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation)) { // Update eye saccades const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; const float AVERAGE_SACCADE_INTERVAL = 6.0f; @@ -95,7 +96,8 @@ void Head::simulate(float deltaTime) { const float BLINK_START_VARIABILITY = 0.25f; const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; - if (getHasProceduralBlinkFaceMovement()) { + if (getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation)) { // handle automatic blinks // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; @@ -146,7 +148,8 @@ void Head::simulate(float deltaTime) { } // use data to update fake Faceshift blendshape coefficients - if (getHasAudioEnabledFaceMovement()) { + if (getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) { // Update audio attack data for facial animation (eyebrows and mouth) float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz _audioAttack = audioAttackAveragingRate * _audioAttack + @@ -178,7 +181,8 @@ void Head::simulate(float deltaTime) { _mouth4, _transientBlendshapeCoefficients); - if (getHasProceduralEyeFaceMovement()) { + if (getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation)) { // This controls two things, the eye brow and the upper eye lid, it is driven by the vertical up/down angle of the // eyes relative to the head. This is to try to help prevent sleepy eyes/crazy eyes. applyEyelidOffset(getOrientation()); diff --git a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp index e0fed08955..b52a68f066 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp @@ -111,6 +111,8 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { Head* head = _owningAvatar->getHead(); + // AJT: blendshapes TODO: RE-enable this. but move into rig? + /* bool eyePosesValid = !head->getHasProceduralEyeMovement(); glm::vec3 lookAt; if (eyePosesValid) { @@ -118,6 +120,8 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { } else { lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); } + */ + glm::vec3 lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); // no need to call Model::updateRig() because otherAvatars get their joint state // copied directly from AvtarData::_jointData (there are no Rig animations to blend) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 520241d020..c5dcfba1c0 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -110,7 +110,6 @@ AvatarData::AvatarData() : _targetScale(1.0f), _handState(0), _keyState(NO_KEY_DOWN), - _hasScriptedBlendshapes(false), _headData(NULL), _errorLogExpiry(0), _owningAvatarMixer(), @@ -156,7 +155,7 @@ float AvatarData::getDomainLimitedScale() const { void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { - if (hasScriptedBlendshapes == _hasScriptedBlendshapes) { + if (hasScriptedBlendshapes == _headData->getHasScriptedBlendshapes()) { return; } if (!hasScriptedBlendshapes) { @@ -165,19 +164,35 @@ void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { // before sending the update, or else it won't send the neutal blendshapes to the receiving clients sendAvatarDataPacket(true); } - _hasScriptedBlendshapes = hasScriptedBlendshapes; + _headData->setHasScriptedBlendshapes(hasScriptedBlendshapes); } -void AvatarData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); +bool AvatarData::getHasScriptedBlendshapes() const { + return _headData->getHasScriptedBlendshapes(); } -void AvatarData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); +void AvatarData::setHasProceduralBlinkFaceMovement(bool value) { + _headData->setProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, value); } -void AvatarData::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); +bool AvatarData::getHasProceduralBlinkFaceMovement() const { + return _headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation); +} + +void AvatarData::setHasProceduralEyeFaceMovement(bool value) { + _headData->setProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, value); +} + +bool AvatarData::getHasProceduralEyeFaceMovement() const { + return _headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation); +} + +void AvatarData::setHasAudioEnabledFaceMovement(bool value) { + _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, value); +} + +bool AvatarData::getHasAudioEnabledFaceMovement() const { + return _headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation); } void AvatarData::setDomainMinimumHeight(float domainMinimumHeight) { @@ -232,9 +247,6 @@ void AvatarData::lazyInitHeadData() const { if (!_headData) { _headData = new HeadData(const_cast(this)); } - if (_hasScriptedBlendshapes) { - _headData->_hasScriptedBlendshapes = true; - } } @@ -555,27 +567,31 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent setAtBit16(flags, HAND_STATE_FINGER_POINTING_BIT); } // face tracker state - if (_headData->_hasScriptedBlendshapes) { + if (_headData->_hasScriptedBlendshapes || _headData->_hasInputDrivenBlendshapes) { setAtBit16(flags, HAS_SCRIPTED_BLENDSHAPES); } // eye tracker state - if (!_headData->_hasProceduralEyeMovement) { - setAtBit16(flags, IS_EYE_TRACKER_CONNECTED); + if (_headData->getProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation)) { + setAtBit16(flags, HAS_PROCEDURAL_EYE_MOVEMENT); } // referential state if (!parentID.isNull()) { setAtBit16(flags, HAS_REFERENTIAL); } // audio face movement - if (_headData->getHasAudioEnabledFaceMovement()) { + if (_headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) { setAtBit16(flags, AUDIO_ENABLED_FACE_MOVEMENT); } // procedural eye face movement - if (_headData->getHasProceduralEyeFaceMovement()) { + if (_headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation)) { setAtBit16(flags, PROCEDURAL_EYE_FACE_MOVEMENT); } // procedural blink face movement - if (_headData->getHasProceduralBlinkFaceMovement()) { + if (_headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation)) { setAtBit16(flags, PROCEDURAL_BLINK_FACE_MOVEMENT); } // avatar collisions enabled @@ -1177,21 +1193,22 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { + (oneAtBit16(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0); auto newHasScriptedBlendshapes = oneAtBit16(bitItems, HAS_SCRIPTED_BLENDSHAPES); - auto newHasntProceduralEyeMovement = oneAtBit16(bitItems, IS_EYE_TRACKER_CONNECTED); - + auto newHasProceduralEyeMovement = oneAtBit16(bitItems, HAS_PROCEDURAL_EYE_MOVEMENT); auto newHasAudioEnabledFaceMovement = oneAtBit16(bitItems, AUDIO_ENABLED_FACE_MOVEMENT); auto newHasProceduralEyeFaceMovement = oneAtBit16(bitItems, PROCEDURAL_EYE_FACE_MOVEMENT); auto newHasProceduralBlinkFaceMovement = oneAtBit16(bitItems, PROCEDURAL_BLINK_FACE_MOVEMENT); + auto newCollideWithOtherAvatars = oneAtBit16(bitItems, COLLIDE_WITH_OTHER_AVATARS); auto newHasPriority = oneAtBit16(bitItems, HAS_HERO_PRIORITY); bool keyStateChanged = (_keyState != newKeyState); bool handStateChanged = (_handState != newHandState); - bool faceStateChanged = (_headData->_hasScriptedBlendshapes != newHasScriptedBlendshapes); - bool eyeStateChanged = (_headData->_hasProceduralEyeMovement == newHasntProceduralEyeMovement); - bool audioEnableFaceMovementChanged = (_headData->getHasAudioEnabledFaceMovement() != newHasAudioEnabledFaceMovement); - bool proceduralEyeFaceMovementChanged = (_headData->getHasProceduralEyeFaceMovement() != newHasProceduralEyeFaceMovement); - bool proceduralBlinkFaceMovementChanged = (_headData->getHasProceduralBlinkFaceMovement() != newHasProceduralBlinkFaceMovement); + bool faceStateChanged = (_headData->getHasScriptedBlendshapes() != newHasScriptedBlendshapes); + + bool eyeStateChanged = (_headData->getProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation) != newHasProceduralEyeMovement); + bool audioEnableFaceMovementChanged = (_headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) != newHasAudioEnabledFaceMovement); + bool proceduralEyeFaceMovementChanged = (_headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) != newHasProceduralEyeFaceMovement); + bool proceduralBlinkFaceMovementChanged = (_headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) != newHasProceduralBlinkFaceMovement); bool collideWithOtherAvatarsChanged = (_collideWithOtherAvatars != newCollideWithOtherAvatars); bool hasPriorityChanged = (getHasPriority() != newHasPriority); bool somethingChanged = keyStateChanged || handStateChanged || faceStateChanged || eyeStateChanged || audioEnableFaceMovementChanged || @@ -1200,11 +1217,11 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _keyState = newKeyState; _handState = newHandState; - _headData->_hasScriptedBlendshapes = newHasScriptedBlendshapes; - _headData->setHasProceduralEyeMovement(!newHasntProceduralEyeMovement); - _headData->setHasAudioEnabledFaceMovement(newHasAudioEnabledFaceMovement); - _headData->setHasProceduralEyeFaceMovement(newHasProceduralEyeFaceMovement); - _headData->setHasProceduralBlinkFaceMovement(newHasProceduralBlinkFaceMovement); + _headData->setHasScriptedBlendshapes(newHasScriptedBlendshapes); + _headData->setProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation, newHasProceduralEyeMovement); + _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, newHasAudioEnabledFaceMovement); + _headData->setProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, newHasProceduralEyeFaceMovement); + _headData->setProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, newHasProceduralBlinkFaceMovement); _collideWithOtherAvatars = newCollideWithOtherAvatars; setHasPriorityWithoutTimestampReset(newHasPriority); @@ -1289,7 +1306,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { sourceBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); PACKET_READ_CHECK(FaceTrackerCoefficients, coefficientsSize); - _headData->_blendshapeCoefficients.resize(numCoefficients); // make sure there's room for the copy! + _headData->_blendshapeCoefficients.resize(std::min(numCoefficients, (int)Blendshapes::BlendshapeCount)); // make sure there's room for the copy! //only copy the blendshapes to headData, not the procedural face info memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, coefficientsSize); sourceBuffer += coefficientsSize; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 6bed5fa9db..53903412d0 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -107,7 +107,7 @@ const quint32 AVATAR_MOTION_SCRIPTABLE_BITS = const int KEY_STATE_START_BIT = 0; // 1st and 2nd bits (UNUSED) const int HAND_STATE_START_BIT = 2; // 3rd and 4th bits (UNUSED) const int HAS_SCRIPTED_BLENDSHAPES = 4; // 5th bit -const int IS_EYE_TRACKER_CONNECTED = 5; // 6th bit (was CHAT_CIRCLING) +const int HAS_PROCEDURAL_EYE_MOVEMENT = 5; // 6th bit const int HAS_REFERENTIAL = 6; // 7th bit const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit (UNUSED) const int AUDIO_ENABLED_FACE_MOVEMENT = 8; // 9th bit @@ -703,13 +703,13 @@ public: float getDomainLimitedScale() const; void setHasScriptedBlendshapes(bool hasScriptedBlendshapes); - bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } + bool getHasScriptedBlendshapes() const; void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasProceduralBlinkFaceMovement() const { return _headData->getHasProceduralBlinkFaceMovement(); } + bool getHasProceduralBlinkFaceMovement() const; void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralEyeFaceMovement() const { return _headData->getHasProceduralEyeFaceMovement(); } + bool getHasProceduralEyeFaceMovement() const; void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasAudioEnabledFaceMovement() const { return _headData->getHasAudioEnabledFaceMovement(); } + bool getHasAudioEnabledFaceMovement() const; /**jsdoc * Gets the minimum scale allowed for this avatar in the current domain. @@ -1716,7 +1716,6 @@ protected: // key state KeyState _keyState; - bool _hasScriptedBlendshapes; bool _hasNewJointData { true }; // set in AvatarData, cleared in Avatar mutable HeadData* _headData { nullptr }; diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 561f2c798a..f37b88c135 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -27,11 +27,10 @@ HeadData::HeadData(AvatarData* owningAvatar) : _basePitch(0.0f), _baseRoll(0.0f), _lookAtPosition(0.0f, 0.0f, 0.0f), - _blendshapeCoefficients(QVector(0, 0.0f)), - _transientBlendshapeCoefficients(QVector(0, 0.0f)), - _summedBlendshapeCoefficients(QVector(0, 0.0f)), _owningAvatar(owningAvatar) { + _userProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, true); + _suppressProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, false); computeBlendshapesLookupMap(); } @@ -102,7 +101,7 @@ const QVector& HeadData::getSummedBlendshapeCoefficients() { void HeadData::setBlendshape(QString name, float val) { - //Check to see if the named blendshape exists, and then set its value if it does + // Check to see if the named blendshape exists, and then set its value if it does auto it = _blendshapeLookupMap.find(name); if (it != _blendshapeLookupMap.end()) { if (_blendshapeCoefficients.size() <= it.value()) { @@ -112,6 +111,19 @@ void HeadData::setBlendshape(QString name, float val) { _transientBlendshapeCoefficients.resize(it.value() + 1); } _blendshapeCoefficients[it.value()] = val; + } else { + // check to see if this is a legacy blendshape that is present in + // ARKit blendshapes but is split. i.e. has left and right halfs. + if (name == "LipsUpperUp") { + _blendshapeCoefficients[(int)Blendshapes::MouthUpperUp_L] = val; + _blendshapeCoefficients[(int)Blendshapes::MouthUpperUp_R] = val; + } else if (name == "LipsLowerDown") { + _blendshapeCoefficients[(int)Blendshapes::MouthLowerDown_L] = val; + _blendshapeCoefficients[(int)Blendshapes::MouthLowerDown_R] = val; + } else if (name == "Sneer") { + _blendshapeCoefficients[(int)Blendshapes::NoseSneer_L] = val; + _blendshapeCoefficients[(int)Blendshapes::NoseSneer_R] = val; + } } } @@ -197,38 +209,34 @@ void HeadData::fromJson(const QJsonObject& json) { } } -bool HeadData::getHasProceduralEyeFaceMovement() const { - return _hasProceduralEyeFaceMovement; +bool HeadData::getProceduralAnimationFlag(ProceduralAnimationType type) const { + return _userProceduralAnimationFlags[(int)type]; } -void HeadData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _hasProceduralEyeFaceMovement = hasProceduralEyeFaceMovement; +void HeadData::setProceduralAnimationFlag(ProceduralAnimationType type, bool value) { + _userProceduralAnimationFlags[(int)type] = value; } -bool HeadData::getHasProceduralBlinkFaceMovement() const { - return _hasProceduralBlinkFaceMovement; +bool HeadData::getSuppressProceduralAnimationFlag(ProceduralAnimationType type) const { + return _suppressProceduralAnimationFlags[(int)type]; } -void HeadData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _hasProceduralBlinkFaceMovement = hasProceduralBlinkFaceMovement; +void HeadData::setSuppressProceduralAnimationFlag(ProceduralAnimationType type, bool value) { + _suppressProceduralAnimationFlags[(int)type] = value; } -bool HeadData::getHasAudioEnabledFaceMovement() const { - return _hasAudioEnabledFaceMovement; -} - -void HeadData::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _hasAudioEnabledFaceMovement = hasAudioEnabledFaceMovement; -} - -bool HeadData::getHasProceduralEyeMovement() const { - return _hasProceduralEyeMovement; -} - -void HeadData::setHasProceduralEyeMovement(bool hasProceduralEyeMovement) { - _hasProceduralEyeMovement = hasProceduralEyeMovement; +bool HeadData::getHasScriptedBlendshapes() const { + return _hasScriptedBlendshapes; } void HeadData::setHasScriptedBlendshapes(bool value) { _hasScriptedBlendshapes = value; } + +bool HeadData::getHasInputDrivenBlendshapes() const { + return _hasInputDrivenBlendshapes; +} + +void HeadData::setHasInputDrivenBlendshapes(bool value) { + _hasInputDrivenBlendshapes = value; +} diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index c6bcc2599b..9652792512 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -72,17 +72,29 @@ public: } bool lookAtPositionChangedSince(quint64 time) { return _lookAtPositionChanged >= time; } - bool getHasProceduralEyeFaceMovement() const; - void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralBlinkFaceMovement() const; - void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasAudioEnabledFaceMovement() const; - void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasProceduralEyeMovement() const; - void setHasProceduralEyeMovement(bool hasProceduralEyeMovement); + enum ProceduralAnimationType { + AudioProceduralBlendshapeAnimation = 0, + BlinkProceduralBlendshapeAnimation, + LidAdjustmentProceduralBlendshapeAnimation, + SaccadeProceduralEyeJointAnimation, + ProceduralAnimaitonTypeCount, + }; + // called by scripts to enable or disable procedural blendshape or eye joint animations. + bool getProceduralAnimationFlag(ProceduralAnimationType type) const; + void setProceduralAnimationFlag(ProceduralAnimationType type, bool value); + + // called by c++ to suppress, i.e. temporarily disable a procedural animation. + bool getSuppressProceduralAnimationFlag(ProceduralAnimationType flag) const; + void setSuppressProceduralAnimationFlag(ProceduralAnimationType flag, bool value); + + // called by scripts to enable/disable manual adjustment of blendshapes void setHasScriptedBlendshapes(bool value); - bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } + bool getHasScriptedBlendshapes() const; + + // called by C++ code to denote the presence of manually driven blendshapes. + void setHasInputDrivenBlendshapes(bool value); + bool getHasInputDrivenBlendshapes() const; friend class AvatarData; @@ -98,21 +110,20 @@ protected: glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; - bool _hasAudioEnabledFaceMovement { true }; - bool _hasProceduralBlinkFaceMovement { true }; - bool _hasProceduralEyeFaceMovement { true }; - bool _hasProceduralEyeMovement { true }; + std::vector _userProceduralAnimationFlags; + std::vector _suppressProceduralAnimationFlags; bool _hasScriptedBlendshapes { false }; + bool _hasInputDrivenBlendshapes { false }; float _leftEyeBlink { 0.0f }; float _rightEyeBlink { 0.0f }; float _averageLoudness { 0.0f }; float _browAudioLift { 0.0f }; - QVector _blendshapeCoefficients; - QVector _transientBlendshapeCoefficients; - QVector _summedBlendshapeCoefficients; + QVector _blendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; + QVector _transientBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; + QVector _summedBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; QMap _blendshapeLookupMap; AvatarData* _owningAvatar; diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index e1ad17aafa..1a5c73a1d8 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -185,70 +185,70 @@ enum class Action { RIGHT_EYE, // AJT: blendshapes - EyeBlink_L, - EyeBlink_R, - EyeSquint_L, - EyeSquint_R, - EyeDown_L, - EyeDown_R, - EyeIn_L, - EyeIn_R, - EyeOpen_L, - EyeOpen_R, - EyeOut_L, - EyeOut_R, - EyeUp_L, - EyeUp_R, - BrowsD_L, - BrowsD_R, - BrowsU_C, - BrowsU_L, - BrowsU_R, - JawFwd, - JawLeft, - JawOpen, - JawRight, - MouthLeft, - MouthRight, - MouthFrown_L, - MouthFrown_R, - MouthSmile_L, - MouthSmile_R, - MouthDimple_L, - MouthDimple_R, - LipsStretch_L, - LipsStretch_R, - LipsUpperClose, - LipsLowerClose, - LipsUpperOpen, - LipsLowerOpen, - LipsFunnel, - LipsPucker, - Puff, - CheekSquint_L, - CheekSquint_R, - LipsTogether, - MouthUpperUp_L, - MouthUpperUp_R, - MouthLowerDown_L, - MouthLowerDown_R, - MouthPress_L, - MouthPress_R, - MouthShrugLower, - MouthShrugUpper, - NoseSneer_L, - NoseSneer_R, - TongueOut, - UserBlendshape0, - UserBlendshape1, - UserBlendshape2, - UserBlendshape3, - UserBlendshape4, - UserBlendshape5, - UserBlendshape6, - UserBlendshape7, - UserBlendshape8, - UserBlendshape9, + EYEBLINK_L, + EYEBLINK_R, + EYESQUINT_L, + EYESQUINT_R, + EYEDOWN_L, + EYEDOWN_R, + EYEIN_L, + EYEIN_R, + EYEOPEN_L, + EYEOPEN_R, + EYEOUT_L, + EYEOUT_R, + EYEUP_L, + EYEUP_R, + BROWSD_L, + BROWSD_R, + BROWSU_C, + BROWSU_L, + BROWSU_R, + JAWFWD, + JAWLEFT, + JAWOPEN, + JAWRIGHT, + MOUTHLEFT, + MOUTHRIGHT, + MOUTHFROWN_L, + MOUTHFROWN_R, + MOUTHSMILE_L, + MOUTHSMILE_R, + MOUTHDIMPLE_L, + MOUTHDIMPLE_R, + LIPSSTRETCH_L, + LIPSSTRETCH_R, + LIPSUPPERCLOSE, + LIPSLOWERCLOSE, + LIPSUPPEROPEN, + LIPSLOWEROPEN, + LIPSFUNNEL, + LIPSPUCKER, + PUFF, + CHEEKSQUINT_L, + CHEEKSQUINT_R, + LIPSTOGETHER, + MOUTHUPPERUP_L, + MOUTHUPPERUP_R, + MOUTHLOWERDOWN_L, + MOUTHLOWERDOWN_R, + MOUTHPRESS_L, + MOUTHPRESS_R, + MOUTHSHRUGLOWER, + MOUTHSHRUGUPPER, + NOSESNEER_L, + NOSESNEER_R, + TONGUEOUT, + USERBLENDSHAPE0, + USERBLENDSHAPE1, + USERBLENDSHAPE2, + USERBLENDSHAPE3, + USERBLENDSHAPE4, + USERBLENDSHAPE5, + USERBLENDSHAPE6, + USERBLENDSHAPE7, + USERBLENDSHAPE8, + USERBLENDSHAPE9, NUM_ACTIONS }; diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index 2af822ffc2..55bcf0e36a 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -92,8 +92,70 @@ namespace controller { RIGHT_GRIP, // AJT: blendshapes - LEFT_EYE_BLINK, - RIGHT_EYE_BLINK, + EYEBLINK_L, + EYEBLINK_R, + EYESQUINT_L, + EYESQUINT_R, + EYEDOWN_L, + EYEDOWN_R, + EYEIN_L, + EYEIN_R, + EYEOPEN_L, + EYEOPEN_R, + EYEOUT_L, + EYEOUT_R, + EYEUP_L, + EYEUP_R, + BROWSD_L, + BROWSD_R, + BROWSU_C, + BROWSU_L, + BROWSU_R, + JAWFWD, + JAWLEFT, + JAWOPEN, + JAWRIGHT, + MOUTHLEFT, + MOUTHRIGHT, + MOUTHFROWN_L, + MOUTHFROWN_R, + MOUTHSMILE_L, + MOUTHSMILE_R, + MOUTHDIMPLE_L, + MOUTHDIMPLE_R, + LIPSSTRETCH_L, + LIPSSTRETCH_R, + LIPSUPPERCLOSE, + LIPSLOWERCLOSE, + LIPSUPPEROPEN, + LIPSLOWEROPEN, + LIPSFUNNEL, + LIPSPUCKER, + PUFF, + CHEEKSQUINT_L, + CHEEKSQUINT_R, + LIPSTOGETHER, + MOUTHUPPERUP_L, + MOUTHUPPERUP_R, + MOUTHLOWERDOWN_L, + MOUTHLOWERDOWN_R, + MOUTHPRESS_L, + MOUTHPRESS_R, + MOUTHSHRUGLOWER, + MOUTHSHRUGUPPER, + NOSESNEER_L, + NOSESNEER_R, + TONGUEOUT, + USERBLENDSHAPE0, + USERBLENDSHAPE1, + USERBLENDSHAPE2, + USERBLENDSHAPE3, + USERBLENDSHAPE4, + USERBLENDSHAPE5, + USERBLENDSHAPE6, + USERBLENDSHAPE7, + USERBLENDSHAPE8, + USERBLENDSHAPE9, NUM_STANDARD_AXES, LZ = LT, diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index ed68fe89dc..87bd7941d3 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -38,10 +38,10 @@ PacketVersion versionForPacketType(PacketType packetType) { return static_cast(EntityQueryPacketVersion::ConicalFrustums); case PacketType::AvatarIdentity: case PacketType::AvatarData: - return static_cast(AvatarMixerPacketVersion::SendVerificationFailed); + return static_cast(AvatarMixerPacketVersion::ARKitBlendshapes); case PacketType::BulkAvatarData: case PacketType::KillAvatar: - return static_cast(AvatarMixerPacketVersion::SendVerificationFailed); + return static_cast(AvatarMixerPacketVersion::ARKitBlendshapes); case PacketType::MessagesData: return static_cast(MessageDataVersion::TextOrBinaryData); // ICE packets diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 52a8c8db16..fbf575065e 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -339,7 +339,8 @@ enum class AvatarMixerPacketVersion : PacketVersion { SendMaxTranslationDimension, FBXJointOrderChange, HandControllerSection, - SendVerificationFailed + SendVerificationFailed, + ARKitBlendshapes }; enum class DomainConnectRequestVersion : PacketVersion { diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 97b459fa0e..63e99f465f 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -91,7 +91,7 @@ enum class LegacyBlendshpaes : int { ChinUpperRaise, // not in ARKit Sneer, // split in ARKit LegacyBlendshapeCount -} +}; // NEW in ARKit // * LipsTogether From 24b43749671c9934bbd35f6b244f588f9a421c6c Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 18 Oct 2019 09:15:56 +1300 Subject: [PATCH 09/33] AudioScope JSDoc --- interface/src/audio/AudioScope.h | 84 ++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 27 deletions(-) diff --git a/interface/src/audio/AudioScope.h b/interface/src/audio/AudioScope.h index 912e337670..fcc9b21efe 100644 --- a/interface/src/audio/AudioScope.h +++ b/interface/src/audio/AudioScope.h @@ -26,19 +26,22 @@ class AudioScope : public QObject, public Dependency { SINGLETON_DEPENDENCY /**jsdoc - * The AudioScope API helps control the Audio Scope features in Interface + * The AudioScope API provides facilities for an audio scope. + * * @namespace AudioScope * + * @deprecated This API doesn't work properly. It is deprecated and will be removed. + * * @hifi-interface * @hifi-client-entity * @hifi-avatar * - * @property {number} scopeInput Read-only. - * @property {number} scopeOutputLeft Read-only. - * @property {number} scopeOutputRight Read-only. - * @property {number} triggerInput Read-only. - * @property {number} triggerOutputLeft Read-only. - * @property {number} triggerOutputRight Read-only. + * @property {number[]} scopeInput - Scope input. Read-only. + * @property {number[]} scopeOutputLeft - Scope left output. Read-only. + * @property {number[]} scopeOutputRight - Scope right output. Read-only. + * @property {number[]} triggerInput - Trigger input. Read-only. + * @property {number[]} triggerOutputLeft - Trigger left output. Read-only. + * @property {number[]} triggerOutputRight - Trigger right output. Read-only. */ Q_PROPERTY(QVector scopeInput READ getScopeInput) @@ -58,159 +61,186 @@ public: public slots: /**jsdoc + * Toggle. * @function AudioScope.toggle */ void toggle() { setVisible(!_isEnabled); } /**jsdoc + * Set visible. * @function AudioScope.setVisible - * @param {boolean} visible + * @param {boolean} visible - Visible. */ void setVisible(bool visible); /**jsdoc + * Get visible. * @function AudioScope.getVisible - * @returns {boolean} + * @returns {boolean} Visible. */ bool getVisible() const { return _isEnabled; } /**jsdoc + * Toggle pause. * @function AudioScope.togglePause */ void togglePause() { setPause(!_isPaused); } /**jsdoc + * Set pause. * @function AudioScope.setPause - * @param {boolean} paused + * @param {boolean} pause - Pause. */ void setPause(bool paused) { _isPaused = paused; emit pauseChanged(); } /**jsdoc + * Get pause. * @function AudioScope.getPause - * @returns {boolean} + * @returns {boolean} Pause. */ bool getPause() { return _isPaused; } /**jsdoc + * Toggle trigger. * @function AudioScope.toggleTrigger */ void toggleTrigger() { _autoTrigger = !_autoTrigger; } /**jsdoc + * Get auto trigger. * @function AudioScope.getAutoTrigger - * @returns {boolean} + * @returns {boolean} Auto trigger. */ bool getAutoTrigger() { return _autoTrigger; } /**jsdoc + * Set auto trigger. * @function AudioScope.setAutoTrigger - * @param {boolean} autoTrigger + * @param {boolean} autoTrigger - Auto trigger. */ void setAutoTrigger(bool autoTrigger) { _isTriggered = false; _autoTrigger = autoTrigger; } /**jsdoc + * Set trigger values. * @function AudioScope.setTriggerValues - * @param {number} x - * @param {number} y + * @param {number} x - X. + * @param {number} y - Y. */ void setTriggerValues(int x, int y) { _triggerValues.x = x; _triggerValues.y = y; } /**jsdoc + * Set triggered. * @function AudioScope.setTriggered - * @param {boolean} triggered + * @param {boolean} triggered - Triggered. */ void setTriggered(bool triggered) { _isTriggered = triggered; } /**jsdoc + * Get triggered. * @function AudioScope.getTriggered - * @returns {boolean} + * @returns {boolean} Triggered. */ bool getTriggered() { return _isTriggered; } /**jsdoc + * Get frames per second. * @function AudioScope.getFramesPerSecond - * @returns {number} + * @returns {number} Frames per second. */ float getFramesPerSecond(); /**jsdoc + * Get frames per scope. * @function AudioScope.getFramesPerScope - * @returns {number} + * @returns {number} Frames per scope. */ int getFramesPerScope() { return _framesPerScope; } /**jsdoc + * Select five frames audio scope. * @function AudioScope.selectAudioScopeFiveFrames */ void selectAudioScopeFiveFrames(); /**jsdoc + * Select twenty frames audio scope. * @function AudioScope.selectAudioScopeTwentyFrames */ void selectAudioScopeTwentyFrames(); /**jsdoc + * Select fifty frames audio scope. * @function AudioScope.selectAudioScopeFiftyFrames */ void selectAudioScopeFiftyFrames(); /**jsdoc + * Get scope input. * @function AudioScope.getScopeInput - * @returns {number[]} + * @returns {number[]} Scope input. */ QVector getScopeInput() { return _scopeInputData; }; /**jsdoc + * Get scope left output. * @function AudioScope.getScopeOutputLeft - * @returns {number[]} + * @returns {number[]} Scope left output. */ QVector getScopeOutputLeft() { return _scopeOutputLeftData; }; /**jsdoc + * Get scope right output. * @function AudioScope.getScopeOutputRight - * @returns {number[]} + * @returns {number[]} Scope right output. */ QVector getScopeOutputRight() { return _scopeOutputRightData; }; /**jsdoc + * Get trigger input. * @function AudioScope.getTriggerInput - * @returns {number[]} + * @returns {number[]} Trigger input. */ QVector getTriggerInput() { return _triggerInputData; }; /**jsdoc + * Get left trigger output. * @function AudioScope.getTriggerOutputLeft - * @returns {number[]} + * @returns {number[]} Left trigger output. */ QVector getTriggerOutputLeft() { return _triggerOutputLeftData; }; /**jsdoc + * Get right trigger output. * @function AudioScope.getTriggerOutputRight - * @returns {number[]} + * @returns {number[]} Right trigger output. */ QVector getTriggerOutputRight() { return _triggerOutputRightData; }; /**jsdoc + * Set local echo. * @function AudioScope.setLocalEcho - * @parm {boolean} localEcho + * @parm {boolean} localEcho - Local echo. */ void setLocalEcho(bool localEcho); /**jsdoc + * Set server echo. * @function AudioScope.setServerEcho - * @parm {boolean} serverEcho + * @parm {boolean} serverEcho - Server echo. */ void setServerEcho(bool serverEcho); signals: /**jsdoc + * Triggered when pause changes. * @function AudioScope.pauseChanged * @returns {Signal} */ void pauseChanged(); /**jsdoc + * Triggered when scoep is triggered. * @function AudioScope.triggered * @returns {Signal} */ From fa9bf4d51ca8996ef3f5f88fb45ac7b94ab301a7 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 18 Oct 2019 09:17:03 +1300 Subject: [PATCH 10/33] JSDoc fixes noticed in passing --- interface/src/avatar/MyAvatar.h | 2 +- interface/src/raypick/LaserPointerScriptingInterface.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 0f139ddbff..c93645331f 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -275,7 +275,7 @@ class MyAvatar : public Avatar { * @property {number} analogPlusSprintSpeed - The sprint (run) speed of your avatar for the "AnalogPlus" control scheme. * @property {MyAvatar.SitStandModelType} userRecenterModel - Controls avatar leaning and recentering behavior. * @property {number} isInSittingState - true if the user wearing the HMD is determined to be sitting - * (avatar leaning is disabled, recenntering is enabled), false if the user wearing the HMD is + * (avatar leaning is disabled, recentering is enabled), false if the user wearing the HMD is * determined to be standing (avatar leaning is enabled, and avatar recenters if it leans too far). * If userRecenterModel == 2 (i.e., auto) the property value automatically updates as the user sits * or stands, unless isSitStandStateLocked == true. Setting the property value overrides the current diff --git a/interface/src/raypick/LaserPointerScriptingInterface.h b/interface/src/raypick/LaserPointerScriptingInterface.h index 6c5ce0dbaf..5745e29e69 100644 --- a/interface/src/raypick/LaserPointerScriptingInterface.h +++ b/interface/src/raypick/LaserPointerScriptingInterface.h @@ -25,10 +25,10 @@ class LaserPointerScriptingInterface : public QObject, public Dependency { * represent objects for repeatedly calculating ray intersections with avatars, entities, and overlays. Ray pointers can also * be configured to generate events on entities and overlays intersected. * - *

Deprecated: This API is deprecated. Use {@link Pointers} instead. - * * @namespace LaserPointers * + * @deprecated This API is deprecated and will be removed. Use {@link Pointers} instead. + * * @hifi-interface * @hifi-client-entity * @hifi-avatar From 92f1b59cc8e555e0594cad64eb5151a3378f888e Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 18 Oct 2019 16:30:32 -0700 Subject: [PATCH 11/33] Full legacy API and FST support --- .../src/avatars/ScriptableAvatar.cpp | 12 ----- .../src/avatars/ScriptableAvatar.h | 7 --- libraries/avatars/src/HeadData.cpp | 3 ++ libraries/avatars/src/HeadData.h | 6 +-- .../controllers/src/controllers/Actions.cpp | 2 +- .../controllers/src/controllers/Actions.h | 2 +- .../src/controllers/StandardController.cpp | 2 +- .../src/controllers/StandardControls.h | 2 +- libraries/fbx/src/FSTReader.cpp | 50 +++++++++++++++++-- libraries/render-utils/src/Model.cpp | 2 + libraries/shared/src/BlendshapeConstants.h | 6 +-- 11 files changed, 60 insertions(+), 34 deletions(-) diff --git a/assignment-client/src/avatars/ScriptableAvatar.cpp b/assignment-client/src/avatars/ScriptableAvatar.cpp index 044ab86942..54eb499be1 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.cpp +++ b/assignment-client/src/avatars/ScriptableAvatar.cpp @@ -279,18 +279,6 @@ void ScriptableAvatar::setJointMappingsFromNetworkReply() { networkReply->deleteLater(); } -void ScriptableAvatar::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); -} - -void ScriptableAvatar::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); -} - -void ScriptableAvatar::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); -} - AvatarEntityMap ScriptableAvatar::getAvatarEntityData() const { // DANGER: Now that we store the AvatarEntityData in packed format this call is potentially Very Expensive! // Avoid calling this method if possible. diff --git a/assignment-client/src/avatars/ScriptableAvatar.h b/assignment-client/src/avatars/ScriptableAvatar.h index fc796b418f..f2f5a1e6f4 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.h +++ b/assignment-client/src/avatars/ScriptableAvatar.h @@ -153,13 +153,6 @@ public: virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking = false) override; - void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasProceduralBlinkFaceMovement() const override { return _headData->getHasProceduralBlinkFaceMovement(); } - void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralEyeFaceMovement() const override { return _headData->getHasProceduralEyeFaceMovement(); } - void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasAudioEnabledFaceMovement() const override { return _headData->getHasAudioEnabledFaceMovement(); } - /**jsdoc * Gets details of all avatar entities. *

Warning: Potentially an expensive call. Do not use if possible.

diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index f37b88c135..ff73677bb8 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -27,6 +27,9 @@ HeadData::HeadData(AvatarData* owningAvatar) : _basePitch(0.0f), _baseRoll(0.0f), _lookAtPosition(0.0f, 0.0f, 0.0f), + _blendshapeCoefficients((int)Blendshapes::BlendshapeCount, 0.0f), + _transientBlendshapeCoefficients((int)Blendshapes::BlendshapeCount, 0.0f), + _summedBlendshapeCoefficients((int)Blendshapes::BlendshapeCount, 0.0f), _owningAvatar(owningAvatar) { _userProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, true); diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 9652792512..218ffceaa1 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -121,9 +121,9 @@ protected: float _averageLoudness { 0.0f }; float _browAudioLift { 0.0f }; - QVector _blendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; - QVector _transientBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; - QVector _summedBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; + QVector _blendshapeCoefficients; + QVector _transientBlendshapeCoefficients; + QVector _summedBlendshapeCoefficients; QMap _blendshapeLookupMap; AvatarData* _owningAvatar; diff --git a/libraries/controllers/src/controllers/Actions.cpp b/libraries/controllers/src/controllers/Actions.cpp index 198c342b1d..6dd9a47395 100644 --- a/libraries/controllers/src/controllers/Actions.cpp +++ b/libraries/controllers/src/controllers/Actions.cpp @@ -350,7 +350,7 @@ namespace controller { makePosePair(Action::LEFT_EYE, "LeftEye"), makePosePair(Action::RIGHT_EYE, "RightEye"), - // AJT: blendshapes + // blendshapes makeAxisPair(Action::EYEBLINK_L, "EyeBlink_L"), makeAxisPair(Action::EYEBLINK_R, "EyeBlink_R"), makeAxisPair(Action::EYESQUINT_L, "EyeSquint_L"), diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index 1a5c73a1d8..86821b4633 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -184,7 +184,7 @@ enum class Action { LEFT_EYE, RIGHT_EYE, - // AJT: blendshapes + // blendshapes EYEBLINK_L, EYEBLINK_R, EYESQUINT_L, diff --git a/libraries/controllers/src/controllers/StandardController.cpp b/libraries/controllers/src/controllers/StandardController.cpp index 14697b26d2..168604ee45 100644 --- a/libraries/controllers/src/controllers/StandardController.cpp +++ b/libraries/controllers/src/controllers/StandardController.cpp @@ -356,7 +356,7 @@ Input::NamedVector StandardController::getAvailableInputs() const { makePair(LEFT_EYE, "LeftEye"), makePair(RIGHT_EYE, "RightEye"), - // AJT: blendshapes + // blendshapes makePair(EYEBLINK_L, "EyeBlink_L"), makePair(EYEBLINK_R, "EyeBlink_R"), makePair(EYESQUINT_L, "EyeSquint_L"), diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index 55bcf0e36a..f99072af7c 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -91,7 +91,7 @@ namespace controller { LEFT_GRIP, RIGHT_GRIP, - // AJT: blendshapes + // blendshapes EYEBLINK_L, EYEBLINK_R, EYESQUINT_L, diff --git a/libraries/fbx/src/FSTReader.cpp b/libraries/fbx/src/FSTReader.cpp index 41b660f722..2835151bfe 100644 --- a/libraries/fbx/src/FSTReader.cpp +++ b/libraries/fbx/src/FSTReader.cpp @@ -21,7 +21,7 @@ QVariantHash FSTReader::parseMapping(QIODevice* device) { QVariantHash properties; - + QByteArray line; while (!(line = device->readLine()).isEmpty()) { if ((line = line.trimmed()).startsWith('#')) { @@ -34,12 +34,10 @@ QVariantHash FSTReader::parseMapping(QIODevice* device) { QByteArray name = sections.at(0).trimmed(); if (sections.size() == 2) { properties.insertMulti(name, sections.at(1).trimmed()); - } else if (sections.size() == 3) { QVariantHash heading = properties.value(name).toHash(); heading.insertMulti(sections.at(1).trimmed(), sections.at(2).trimmed()); properties.insert(name, heading); - } else if (sections.size() >= 4) { QVariantHash heading = properties.value(name).toHash(); QVariantList contents; @@ -50,14 +48,56 @@ QVariantHash FSTReader::parseMapping(QIODevice* device) { properties.insert(name, heading); } } - + return properties; } +static void removeBlendshape(QVariantHash& bs, const QString& key) { + if (bs.contains(key)) { + bs.remove(key); + } +} + +static void splitBlendshapes(QVariantHash& bs, const QString& key, const QString& leftKey, const QString& rightKey) { + if (bs.contains(key) && !(bs.contains(leftKey) || bs.contains(rightKey))) { + // key has been split into leftKey and rightKey blendshapes + QVariantList origShapes = bs.values(key); + QVariantList halfShapes; + for (int i = 0; i < origShapes.size(); i++) { + QVariantList origShape = origShapes[i].toList(); + QVariantList halfShape; + halfShape.append(origShape[0]); + halfShape.append(QVariant(0.5f * origShape[1].toFloat())); + bs.insertMulti(leftKey, halfShape); + bs.insertMulti(rightKey, halfShape); + } + } +} + +// convert legacy blendshapes to arkit blendshapes +static void fixUpLegacyBlendshapes(QVariantHash& properties) { + QVariantHash bs = properties.value("bs").toHash(); + + // These blendshapes have no ARKit equivalent, so we remove them. + removeBlendshape(bs, "JawChew"); + removeBlendshape(bs, "ChinLowerRaise"); + removeBlendshape(bs, "ChinUpperRaise"); + + // These blendshapes are split in ARKit, we replace them with their left and right sides with a weight of 1/2. + splitBlendshapes(bs, "LipsUpperUp", "MouthUpperUp_L", "MouthUpperUp_R"); + splitBlendshapes(bs, "LipsLowerDown", "MouthLowerDown_L", "MouthLowerDown_R"); + splitBlendshapes(bs, "Sneer", "NoseSneer_L", "NoseSneer_R"); + + // re-insert new mutated bs hash into mapping properties. + properties.insert("bs", bs); +} + QVariantHash FSTReader::readMapping(const QByteArray& data) { QBuffer buffer(const_cast(&data)); buffer.open(QIODevice::ReadOnly); - return FSTReader::parseMapping(&buffer); + QVariantHash mapping = FSTReader::parseMapping(&buffer); + fixUpLegacyBlendshapes(mapping); + return mapping; } void FSTReader::writeVariant(QBuffer& buffer, QVariantHash::const_iterator& it) { diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 74cf1ffa39..a7e098e1b7 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -39,6 +39,8 @@ #include "RenderUtilsLogging.h" #include +#include + using namespace std; int nakedModelPointerTypeId = qRegisterMetaType(); diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 63e99f465f..0f934b2056 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -109,10 +109,10 @@ enum class LegacyBlendshpaes : int { // Legacy shapes // * JawChew (not in ARKit) -// * MouthUpperUp (split in ARKit) -// * MouthLowerDown (split in ARKit) +// * LipsUpperUp (split in ARKit) +// * LipsLowerDown (split in ARKit) // * Sneer (split in ARKit) // * ChinLowerRaise (not in ARKit) -// * ChinUpperRase (not in ARKit) +// * ChinUpperRaise (not in ARKit) #endif // hifi_BlendshapeConstants_h From 03f88f696d1c1d8d31ec5649157b9f887f944850 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Mon, 21 Oct 2019 19:37:57 -0700 Subject: [PATCH 12/33] Renamed LipsTogether to MouthClose and bug fixes. --- interface/resources/controllers/standard.json | 2 +- interface/resources/controllers/standard_nomovement.json | 2 +- interface/src/avatar/MyHead.cpp | 2 +- libraries/avatars/src/AvatarData.cpp | 4 ++++ libraries/avatars/src/HeadData.cpp | 4 ++++ libraries/avatars/src/HeadData.h | 1 + libraries/controllers/src/controllers/Actions.cpp | 2 +- libraries/controllers/src/controllers/Actions.h | 2 +- libraries/controllers/src/controllers/StandardController.cpp | 2 +- libraries/controllers/src/controllers/StandardControls.h | 2 +- libraries/shared/src/BlendshapeConstants.cpp | 2 +- libraries/shared/src/BlendshapeConstants.h | 4 ++-- 12 files changed, 19 insertions(+), 10 deletions(-) diff --git a/interface/resources/controllers/standard.json b/interface/resources/controllers/standard.json index 8c84039b86..b6cb4e3e27 100644 --- a/interface/resources/controllers/standard.json +++ b/interface/resources/controllers/standard.json @@ -208,7 +208,7 @@ { "from": "Standard.Puff", "to": "Actions.Puff" }, { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, - { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthClose", "to": "Actions.MouthClose" }, { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, diff --git a/interface/resources/controllers/standard_nomovement.json b/interface/resources/controllers/standard_nomovement.json index e311a5fa62..04c0d2f329 100644 --- a/interface/resources/controllers/standard_nomovement.json +++ b/interface/resources/controllers/standard_nomovement.json @@ -103,7 +103,7 @@ { "from": "Standard.Puff", "to": "Actions.Puff" }, { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, - { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthClose", "to": "Actions.MouthClose" }, { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index f92f8b7218..a0e70a3049 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -64,7 +64,7 @@ static controller::Action blendshapeActions[] = { controller::Action::PUFF, controller::Action::CHEEKSQUINT_L, controller::Action::CHEEKSQUINT_R, - controller::Action::LIPSTOGETHER, + controller::Action::MOUTHCLOSE, controller::Action::MOUTHUPPERUP_L, controller::Action::MOUTHUPPERUP_R, controller::Action::MOUTHLOWERDOWN_L, diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index c5dcfba1c0..87d6c57d59 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1217,6 +1217,10 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _keyState = newKeyState; _handState = newHandState; + if (!newHasScriptedBlendshapes && getHasScriptedBlendshapes()) { + // if scripted blendshapes have just been turned off, slam blendshapes back to zero. + _headData->clearBlendshapeCoefficients(); + } _headData->setHasScriptedBlendshapes(newHasScriptedBlendshapes); _headData->setProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation, newHasProceduralEyeMovement); _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, newHasAudioEnabledFaceMovement); diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index ff73677bb8..93fb0190bb 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -83,6 +83,10 @@ int HeadData::getNumSummedBlendshapeCoefficients() const { return maxSize; } +void HeadData::clearBlendshapeCoefficients() { + _blendshapeCoefficients.fill(0.0f, (int)_blendshapeCoefficients.size()); +} + const QVector& HeadData::getSummedBlendshapeCoefficients() { int maxSize = std::max(_blendshapeCoefficients.size(), _transientBlendshapeCoefficients.size()); if (_summedBlendshapeCoefficients.size() != maxSize) { diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 218ffceaa1..083374b32d 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -62,6 +62,7 @@ public: const QVector& getSummedBlendshapeCoefficients(); int getNumSummedBlendshapeCoefficients() const; void setBlendshapeCoefficients(const QVector& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; } + void clearBlendshapeCoefficients(); const glm::vec3& getLookAtPosition() const { return _lookAtPosition; } void setLookAtPosition(const glm::vec3& lookAtPosition) { diff --git a/libraries/controllers/src/controllers/Actions.cpp b/libraries/controllers/src/controllers/Actions.cpp index 6dd9a47395..36f454b5d0 100644 --- a/libraries/controllers/src/controllers/Actions.cpp +++ b/libraries/controllers/src/controllers/Actions.cpp @@ -393,7 +393,7 @@ namespace controller { makeAxisPair(Action::PUFF, "Puff"), makeAxisPair(Action::CHEEKSQUINT_L, "CheekSquint_L"), makeAxisPair(Action::CHEEKSQUINT_R, "CheekSquint_R"), - makeAxisPair(Action::LIPSTOGETHER, "LipsTogether"), + makeAxisPair(Action::MOUTHCLOSE, "MouthClose"), makeAxisPair(Action::MOUTHUPPERUP_L, "MouthUpperUp_L"), makeAxisPair(Action::MOUTHUPPERUP_R, "MouthUpperUp_R"), makeAxisPair(Action::MOUTHLOWERDOWN_L, "MouthLowerDown_L"), diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index 86821b4633..5c96923dc3 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -227,7 +227,7 @@ enum class Action { PUFF, CHEEKSQUINT_L, CHEEKSQUINT_R, - LIPSTOGETHER, + MOUTHCLOSE, MOUTHUPPERUP_L, MOUTHUPPERUP_R, MOUTHLOWERDOWN_L, diff --git a/libraries/controllers/src/controllers/StandardController.cpp b/libraries/controllers/src/controllers/StandardController.cpp index 168604ee45..936f1c391f 100644 --- a/libraries/controllers/src/controllers/StandardController.cpp +++ b/libraries/controllers/src/controllers/StandardController.cpp @@ -399,7 +399,7 @@ Input::NamedVector StandardController::getAvailableInputs() const { makePair(PUFF, "Puff"), makePair(CHEEKSQUINT_L, "CheekSquint_L"), makePair(CHEEKSQUINT_R, "CheekSquint_R"), - makePair(LIPSTOGETHER, "LipsTogether"), + makePair(MOUTHCLOSE, "MouthClose"), makePair(MOUTHUPPERUP_L, "MouthUpperUp_L"), makePair(MOUTHUPPERUP_R, "MouthUpperUp_R"), makePair(MOUTHLOWERDOWN_L, "MouthLowerDown_L"), diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index f99072af7c..965c095187 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -134,7 +134,7 @@ namespace controller { PUFF, CHEEKSQUINT_L, CHEEKSQUINT_R, - LIPSTOGETHER, + MOUTHCLOSE, MOUTHUPPERUP_L, MOUTHUPPERUP_R, MOUTHLOWERDOWN_L, diff --git a/libraries/shared/src/BlendshapeConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp index 7564c31944..91b68ed8a9 100644 --- a/libraries/shared/src/BlendshapeConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -54,7 +54,7 @@ const char* FACESHIFT_BLENDSHAPES[] = { "Puff", "CheekSquint_L", "CheekSquint_R", - "LipsTogether", + "MouthClose", "MouthUpperUp_L", "MouthUpperUp_R", "MouthLowerDown_L", diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 0f934b2056..8db29856c3 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -58,7 +58,7 @@ enum class Blendshapes : int { Puff, CheekSquint_L, CheekSquint_R, - LipsTogether, + MouthClose, MouthUpperUp_L, MouthUpperUp_R, MouthLowerDown_L, @@ -94,7 +94,7 @@ enum class LegacyBlendshpaes : int { }; // NEW in ARKit -// * LipsTogether +// * MouthClose // * MouthUpperUp_L // * MouthUpperUp_R // * MouthLowerDown_L From 61b9f08fec914aa6a8233a398a545dfaa87e1da0 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 22 Oct 2019 22:19:03 -0700 Subject: [PATCH 13/33] Blendshapes for OtherAvatars should update even if no joints have changes. --- interface/src/avatar/OtherAvatar.cpp | 5 ++++- .../avatars-renderer/src/avatars-renderer/SkeletonModel.cpp | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/OtherAvatar.cpp b/interface/src/avatar/OtherAvatar.cpp index 5673c2443f..50f6369dbe 100755 --- a/interface/src/avatar/OtherAvatar.cpp +++ b/interface/src/avatar/OtherAvatar.cpp @@ -267,6 +267,7 @@ void OtherAvatar::simulate(float deltaTime, bool inView) { _skeletonModel->getRig().computeExternalPoses(rootTransform); _jointDataSimulationRate.increment(); + head->simulate(deltaTime); _skeletonModel->simulate(deltaTime, true); locationChanged(); // joints changed, so if there are any children, update them. @@ -277,9 +278,11 @@ void OtherAvatar::simulate(float deltaTime, bool inView) { headPosition = getWorldPosition(); } head->setPosition(headPosition); + } else { + head->simulate(deltaTime); + _skeletonModel->simulate(deltaTime, false); } head->setScale(getModelScale()); - head->simulate(deltaTime); relayJointDataToChildren(); } else { // a non-full update is still required so that the position, rotation, scale and bounds of the skeletonModel are updated. diff --git a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp index b52a68f066..fc16851942 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp @@ -165,8 +165,9 @@ void SkeletonModel::updateAttitude(const glm::quat& orientation) { // but just before head has been simulated. void SkeletonModel::simulate(float deltaTime, bool fullUpdate) { updateAttitude(_owningAvatar->getWorldOrientation()); + setBlendshapeCoefficients(_owningAvatar->getHead()->getSummedBlendshapeCoefficients()); + if (fullUpdate) { - setBlendshapeCoefficients(_owningAvatar->getHead()->getSummedBlendshapeCoefficients()); Parent::simulate(deltaTime, fullUpdate); From 777f9699192d9e542f06dd61f9e324a1d502b076 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 24 Oct 2019 09:00:29 +1300 Subject: [PATCH 14/33] Typo --- libraries/script-engine/src/RecordingScriptingInterface.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/script-engine/src/RecordingScriptingInterface.h b/libraries/script-engine/src/RecordingScriptingInterface.h index 19db6dc50d..fd9c2d64e6 100644 --- a/libraries/script-engine/src/RecordingScriptingInterface.h +++ b/libraries/script-engine/src/RecordingScriptingInterface.h @@ -265,7 +265,7 @@ public slots: * Gets the default directory that recordings are saved in. * @function Recording.getDefaultRecordingSaveDirectory * @returns {string} The default recording save directory. - * @exampl Report the default save directory. + * @example Report the default save directory. * print("Default save directory: " + Recording.getDefaultRecordingSaveDirectory()); */ QString getDefaultRecordingSaveDirectory(); From 116510225ad6f9590b7d7deaeefe77012be5bf66 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 24 Oct 2019 09:10:26 +1300 Subject: [PATCH 15/33] Typo --- libraries/ui/src/ui/ToolbarScriptingInterface.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ui/src/ui/ToolbarScriptingInterface.h b/libraries/ui/src/ui/ToolbarScriptingInterface.h index a4746e4566..746ba2894e 100644 --- a/libraries/ui/src/ui/ToolbarScriptingInterface.h +++ b/libraries/ui/src/ui/ToolbarScriptingInterface.h @@ -19,7 +19,7 @@ class QQuickItem; -// No JSDoc for ToolbarButtonProxy because oolbarProxy#addButton() doesn't work. +// No JSDoc for ToolbarButtonProxy because ToolbarProxy#addButton() doesn't work. class ToolbarButtonProxy : public QmlWrapper { Q_OBJECT From 423074463dada43a8b70d851f53cf5423193a8bb Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 24 Oct 2019 09:17:13 +1300 Subject: [PATCH 16/33] Typo --- interface/src/audio/AudioScope.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/audio/AudioScope.h b/interface/src/audio/AudioScope.h index fcc9b21efe..26b228e900 100644 --- a/interface/src/audio/AudioScope.h +++ b/interface/src/audio/AudioScope.h @@ -240,7 +240,7 @@ signals: void pauseChanged(); /**jsdoc - * Triggered when scoep is triggered. + * Triggered when scope is triggered. * @function AudioScope.triggered * @returns {Signal} */ From 2591aed686acebf7c599199468a890f04a3bb1fd Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 24 Oct 2019 09:20:24 +1300 Subject: [PATCH 17/33] Exclude AudioScope API from JSDoc output --- tools/jsdoc/plugins/hifi.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jsdoc/plugins/hifi.js b/tools/jsdoc/plugins/hifi.js index 07549530ce..56d2278b50 100644 --- a/tools/jsdoc/plugins/hifi.js +++ b/tools/jsdoc/plugins/hifi.js @@ -26,7 +26,7 @@ exports.handlers = { '../../assignment-client/src/octree', '../../interface/src', '../../interface/src/assets', - '../../interface/src/audio', + //'../../interface/src/audio', Exlude AudioScope API from output. '../../interface/src/avatar', '../../interface/src/commerce', '../../interface/src/devices', From f28d317914448c90f0c2a1697de8832b5dc29975 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 23 Oct 2019 16:43:10 -0700 Subject: [PATCH 18/33] Clean up reamining issues --- interface/src/avatar/MySkeletonModel.cpp | 14 +++++--------- .../src/avatars-renderer/SkeletonModel.cpp | 11 ----------- libraries/avatars/src/AvatarData.cpp | 3 ++- libraries/avatars/src/HeadData.cpp | 9 +-------- 4 files changed, 8 insertions(+), 29 deletions(-) diff --git a/interface/src/avatar/MySkeletonModel.cpp b/interface/src/avatar/MySkeletonModel.cpp index 0d7bd3f3b3..6fe199aaba 100755 --- a/interface/src/avatar/MySkeletonModel.cpp +++ b/interface/src/avatar/MySkeletonModel.cpp @@ -112,23 +112,19 @@ static AnimPose computeHipsInSensorFrame(MyAvatar* myAvatar, bool isFlying) { void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { const HFMModel& hfmModel = getHFMModel(); + MyAvatar* myAvatar = static_cast(_owningAvatar); + assert(myAvatar); + Head* head = _owningAvatar->getHead(); - // AJT: blendshapes TODO: RE-enable this and avoid duplication with - // SkeletonModel::updateRig() - /* - bool eyePosesValid = !head->getHasProceduralEyeMovement(); + bool eyePosesValid = (myAvatar->getControllerPoseInSensorFrame(controller::Action::LEFT_EYE).isValid() || + myAvatar->getControllerPoseInSensorFrame(controller::Action::RIGHT_EYE).isValid()); glm::vec3 lookAt; if (eyePosesValid) { lookAt = head->getLookAtPosition(); // don't apply no-crosseyes code when eyes are being tracked } else { lookAt = avoidCrossedEyes(head->getLookAtPosition()); } - */ - glm::vec3 lookAt = avoidCrossedEyes(head->getLookAtPosition()); - - MyAvatar* myAvatar = static_cast(_owningAvatar); - assert(myAvatar); Rig::ControllerParameters params; diff --git a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp index fc16851942..ccc87c28f3 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp @@ -110,17 +110,6 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { assert(!_owningAvatar->isMyAvatar()); Head* head = _owningAvatar->getHead(); - - // AJT: blendshapes TODO: RE-enable this. but move into rig? - /* - bool eyePosesValid = !head->getHasProceduralEyeMovement(); - glm::vec3 lookAt; - if (eyePosesValid) { - lookAt = head->getLookAtPosition(); // don't apply no-crosseyes code etc when eyes are being tracked - } else { - lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); - } - */ glm::vec3 lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); // no need to call Model::updateRig() because otherAvatars get their joint state diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 87d6c57d59..818902968f 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2637,6 +2637,7 @@ enum class JsonAvatarFrameVersion : int { JointRotationsInAbsoluteFrame, JointDefaultPoseBits, JointUnscaledTranslations, + ARKitBlendshapes }; QJsonValue toJsonValue(const JointData& joint) { @@ -2681,7 +2682,7 @@ void AvatarData::avatarEntityDataToJson(QJsonObject& root) const { QJsonObject AvatarData::toJson() const { QJsonObject root; - root[JSON_AVATAR_VERSION] = (int)JsonAvatarFrameVersion::JointUnscaledTranslations; + root[JSON_AVATAR_VERSION] = (int)JsonAvatarFrameVersion::ARKitBlendshapes; if (!getSkeletonModelURL().isEmpty()) { root[JSON_AVATAR_BODY_MODEL] = getSkeletonModelURL().toString(); diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 93fb0190bb..5e8d5c457f 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -186,14 +186,7 @@ QJsonObject HeadData::toJson() const { void HeadData::fromJson(const QJsonObject& json) { if (json.contains(JSON_AVATAR_HEAD_BLENDSHAPE_COEFFICIENTS)) { auto jsonValue = json[JSON_AVATAR_HEAD_BLENDSHAPE_COEFFICIENTS]; - if (jsonValue.isArray()) { - QVector blendshapeCoefficients; - QJsonArray blendshapeCoefficientsJson = jsonValue.toArray(); - for (const auto& blendshapeCoefficient : blendshapeCoefficientsJson) { - blendshapeCoefficients.push_back((float)blendshapeCoefficient.toDouble()); - } - setBlendshapeCoefficients(blendshapeCoefficients); - } else if (jsonValue.isObject()) { + if (jsonValue.isObject()) { QJsonObject blendshapeCoefficientsJson = jsonValue.toObject(); for (const QString& name : blendshapeCoefficientsJson.keys()) { float value = (float)blendshapeCoefficientsJson[name].toDouble(); From dc7b611e8e8aded9b618868d1dcfef444dbe25eb Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Thu, 24 Oct 2019 09:53:11 -0700 Subject: [PATCH 19/33] documentation and warning fixes --- interface/src/avatar/MyAvatar.cpp | 2 -- interface/src/avatar/MyAvatar.h | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index f1829db8f6..52083af8f2 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -6535,7 +6535,6 @@ void MyAvatar::updateEyesLookAtPosition(float deltaTime) { updateLookAtTargetAvatar(); - bool isLookingAtSomeone = false; glm::vec3 lookAtSpot; const MyHead* myHead = getMyHead(); @@ -6596,7 +6595,6 @@ void MyAvatar::updateEyesLookAtPosition(float deltaTime) { avatar && avatar->getLookAtSnappingEnabled() && getLookAtSnappingEnabled(); if (haveLookAtCandidate && mutualLookAtSnappingEnabled) { // If I am looking at someone else, look directly at one of their eyes - isLookingAtSomeone = true; auto lookingAtHead = avatar->getHead(); const float MAXIMUM_FACE_ANGLE = 65.0f * RADIANS_PER_DEGREE; diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 953b7ef1dc..9197b005be 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -307,7 +307,7 @@ class MyAvatar : public Avatar { * @borrows Avatar.clearAvatarEntity as clearAvatarEntity * @borrows Avatar.hasScriptedBlendshapes as hasScriptedBlendshapes * @borrows Avatar.hasProceduralBlinkFaceMovement as hasProceduralBlinkFaceMovement - * @borrows Avatar.hasEyeFaceMovement as hasEyeFaceMovement + * @borrows Avatar.hasProceduralEyeFaceMovement as hasProceduralEyeFaceMovement * @borrows Avatar.hasAudioEnabledFaceMovement as hasAudioEnabledFaceMovement * @borrows Avatar.setSkeletonModelURL as setSkeletonModelURL * @borrows Avatar.getAttachmentData as getAttachmentData From 9325ea588a1b0fda23bea233eaa64ad924c2c5d1 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Sat, 26 Oct 2019 09:20:51 -0700 Subject: [PATCH 20/33] Re-added removed jsdoc comments for removed AvatarInput jsapi. With a comment that these properties/functions have been removed. --- interface/src/ui/AvatarInputs.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/interface/src/ui/AvatarInputs.h b/interface/src/ui/AvatarInputs.h index 40ee0ea7d4..3b0e57d037 100644 --- a/interface/src/ui/AvatarInputs.h +++ b/interface/src/ui/AvatarInputs.h @@ -32,6 +32,14 @@ class AvatarInputs : public QObject { * @hifi-client-entity * @hifi-avatar * + * @property {boolean} cameraEnabled - true if webcam face tracking is enabled, false if it is + * disabled. + * Read-only. + *

Deprecated: This property is deprecated and has been removed.

+ * @property {boolean} cameraMuted - true if webcam face tracking is muted (temporarily disabled), + * false it if isn't. + * Read-only. + *

Deprecated: This property is deprecated and has been removed.

* @property {boolean} ignoreRadiusEnabled - true if the privacy shield is enabled, false if it * is disabled. * Read-only. @@ -87,6 +95,20 @@ public slots: signals: + /**jsdoc + * Triggered when webcam face tracking is enabled or disabled. + * @deprecated This signal is deprecated and has been removed. + * @function AvatarInputs.cameraEnabledChanged + * @returns {Signal} + */ + + /**jsdoc + * Triggered when webcam face tracking is muted (temporarily disabled) or unmuted. + * @deprecated This signal is deprecated and has been removed. + * @function AvatarInputs.cameraMutedChanged + * @returns {Signal} + */ + /**jsdoc * Triggered when the display mode changes between desktop and HMD. * @function AvatarInputs.isHMDChanged @@ -157,6 +179,12 @@ protected: */ Q_INVOKABLE void resetSensors(); + /**jsdoc + * Toggles the muting (temporary disablement) of webcam face tracking on/off. + *

Deprecated: This function is deprecated and has been removed.

+ * @function AvatarInputs.toggleCameraMute + */ + private: void onAvatarEnteredIgnoreRadius(); void onAvatarLeftIgnoreRadius(); From 70c3bb2748827cffb7bd0f60402a752af31e2c1c Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Tue, 10 Sep 2019 15:38:20 +0300 Subject: [PATCH 21/33] Emit an event when failing to load an avatar. Previously, MyAvatar only emitted an event (onLoadComplete) if the load succeeded. Now it also emits an event (onLoadFailed) if the load failed. --- interface/src/avatar/MyAvatar.cpp | 5 ++++- interface/src/avatar/MyAvatar.h | 9 ++++++++- libraries/animation/src/Rig.cpp | 6 +++++- libraries/animation/src/Rig.h | 1 + 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 6e0bfab69b..4392505fb3 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -349,7 +349,8 @@ MyAvatar::MyAvatar(QThread* thread) : } }); - connect(&(_skeletonModel->getRig()), SIGNAL(onLoadComplete()), this, SIGNAL(onLoadComplete())); + connect(&(_skeletonModel->getRig()), &Rig::onLoadComplete, this, &MyAvatar::onLoadComplete); + connect(&(_skeletonModel->getRig()), &Rig::onLoadFailed, this, &MyAvatar::onLoadFailed); _characterController.setDensity(_density); } @@ -2626,6 +2627,8 @@ void MyAvatar::useFullAvatarURL(const QUrl& fullAvatarURL, const QString& modelN if (urlString.isEmpty() || (fullAvatarURL != getSkeletonModelURL())) { setSkeletonModelURL(fullAvatarURL); UserActivityLogger::getInstance().changedModel("skeleton", urlString); + } else { + emit onLoadComplete(); } } diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 081fd00d5b..4c7f0c4a75 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -2435,10 +2435,17 @@ signals: /**jsdoc * Triggered when the avatar's model finishes loading. * @function MyAvatar.onLoadComplete - * @returns {Signal} + * @returns {Signal} */ void onLoadComplete(); + /**jsdoc + * Triggered when the avatar's model has failed to load. + * @function MyAvatar.onLoadFailed + * @returns {Signal} + */ + void onLoadFailed(); + /**jsdoc * Triggered when your avatar changes from being active to being away. * @function MyAvatar.wentAway diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index fac4e04ce9..fc1885ea2b 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -2353,6 +2353,7 @@ void Rig::initAnimGraph(const QUrl& url) { // abort load if the previous skeleton was deleted. auto sharedSkeletonPtr = weakSkeletonPtr.lock(); if (!sharedSkeletonPtr) { + emit onLoadFailed(); return; } @@ -2386,8 +2387,9 @@ void Rig::initAnimGraph(const QUrl& url) { } emit onLoadComplete(); }); - connect(_animLoader.get(), &AnimNodeLoader::error, [url](int error, QString str) { + connect(_animLoader.get(), &AnimNodeLoader::error, [this, url](int error, QString str) { qCritical(animation) << "Error loading: code = " << error << "str =" << str; + emit onLoadFailed(); }); connect(_networkLoader.get(), &AnimNodeLoader::success, [this, weakSkeletonPtr, networkUrl](AnimNode::Pointer nodeIn) { @@ -2415,6 +2417,8 @@ void Rig::initAnimGraph(const QUrl& url) { connect(_networkLoader.get(), &AnimNodeLoader::error, [networkUrl](int error, QString str) { qCritical(animation) << "Error loading: code = " << error << "str =" << str; }); + } else { + emit onLoadComplete(); } } diff --git a/libraries/animation/src/Rig.h b/libraries/animation/src/Rig.h index 8570ae4441..b2b9ecd5b4 100644 --- a/libraries/animation/src/Rig.h +++ b/libraries/animation/src/Rig.h @@ -260,6 +260,7 @@ public: signals: void onLoadComplete(); + void onLoadFailed(); protected: bool isIndexValid(int index) const { return _animSkeleton && index >= 0 && index < _animSkeleton->getNumJoints(); } From 884cbd7de93370a0d9ed0ec053b87bc1c8475cf9 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 09:26:48 -0700 Subject: [PATCH 22/33] working on moving display plugin lookup to audiodevices --- interface/src/scripting/AudioDevices.cpp | 29 +++++++++- libraries/audio-client/src/AudioClient.cpp | 57 ++++++++----------- libraries/audio-client/src/AudioClient.h | 5 +- .../plugins/src/plugins/PluginManager.cpp | 8 +-- 4 files changed, 58 insertions(+), 41 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index 76c871fa5c..8737cd1296 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include "Application.h" @@ -68,6 +69,21 @@ static QString getTargetDevice(bool hmd, QAudio::Mode mode) { return deviceName; } + +static QString getHmdAudioDeviceName(QAudio::Mode mode) { + foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getAllDisplayPlugins()) { + if (displayPlugin && displayPlugin->isHmd()) { + if (mode == QAudio::AudioInput) { + return displayPlugin->getPreferredAudioInDevice(); + } else { + return displayPlugin->getPreferredAudioOutDevice(); + } + break; + } + } + return QString(); +} + Qt::ItemFlags AudioDeviceList::_flags { Qt::ItemIsSelectable | Qt::ItemIsEnabled }; AudioDeviceList::AudioDeviceList(QAudio::Mode mode) : _mode(mode) { @@ -263,6 +279,18 @@ void AudioDeviceList::onDevicesChanged(const QList& devices bool hmdIsSelected = false; bool desktopIsSelected = false; + //getting hmd mode + if (devices.size() > 0) { + auto mode = devices.first().getMode(); + QString name = getHmdAudioDeviceName(mode); + if (!name.isEmpty()) { + auto client = DependencyManager::get().data(); + QMetaObject::invokeMethod(client, "setHmdAudioName", + Q_ARG(QAudio::Mode, mode), + Q_ARG(const QString&, name)); + } + } + if (!_backupSelectedDesktopDeviceName.isEmpty() && !_backupSelectedHMDDeviceName.isEmpty()) { foreach(const HifiAudioDeviceInfo& deviceInfo, devices) { for (bool isHMD : {false, true}) { @@ -275,7 +303,6 @@ void AudioDeviceList::onDevicesChanged(const QList& devices _selectedDesktopDevice = deviceInfo; backupSelectedDeviceName.clear(); } - } } } diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index fe4fab8415..e966274a9b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -48,7 +48,6 @@ #include #include #include -#include #include "AudioClientLogging.h" #include "AudioLogging.h" @@ -83,38 +82,31 @@ Mutex _recordMutex; QString defaultAudioDeviceName(QAudio::Mode mode); -static QString getHmdAudioDeviceName(QAudio::Mode mode) { - foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getAllDisplayPlugins()) { - if (displayPlugin && displayPlugin->isHmd()) { - if (mode == QAudio::AudioInput) { - return displayPlugin->getPreferredAudioInDevice(); - } else { - return displayPlugin->getPreferredAudioOutDevice(); - } - break; - } +void AudioClient::setHmdAudioName(QAudio::Mode mode, const QString& name) { + if (mode == QAudio::AudioInput) { + _hmdInputName = name; + } else { + _hmdOutputName = name; } - return QString(); } // thread-safe -QList getAvailableDevices(QAudio::Mode mode) { +QList getAvailableDevices(QAudio::Mode mode, const QString& hmdName) { //get hmd device name prior to locking device mutex. in case of shutdown, this thread will be locked and audio client //cannot properly shut down. - QString hmdDeviceName = getHmdAudioDeviceName(mode); QString defDeviceName = defaultAudioDeviceName(mode); // NOTE: availableDevices() clobbers the Qt internal device list Lock lock(_deviceMutex); auto devices = QAudioDeviceInfo::availableDevices(mode); - + HifiAudioDeviceInfo defaultDesktopDevice; QList newDevices; for (auto& device : devices) { newDevices.push_back(HifiAudioDeviceInfo(device, false, mode)); if (device.deviceName() == defDeviceName.trimmed()) { defaultDesktopDevice = HifiAudioDeviceInfo(device, true, mode, HifiAudioDeviceInfo::desktop); - } + } } if (defaultDesktopDevice.getDevice().isNull()) { @@ -123,11 +115,10 @@ QList getAvailableDevices(QAudio::Mode mode) { defaultDesktopDevice = HifiAudioDeviceInfo(devices.first(), true, mode, HifiAudioDeviceInfo::desktop); } newDevices.push_front(defaultDesktopDevice); - - if (!hmdDeviceName.isNull() && !hmdDeviceName.isEmpty()) { + if (!hmdName.isNull()) { HifiAudioDeviceInfo hmdDevice; foreach(auto device, newDevices) { - if (device.getDevice().deviceName() == hmdDeviceName) { + if (device.getDevice().deviceName() == hmdName) { hmdDevice = HifiAudioDeviceInfo(device.getDevice(), true, mode, HifiAudioDeviceInfo::hmd); break; } @@ -149,9 +140,9 @@ void AudioClient::checkDevices() { return; } - auto inputDevices = getAvailableDevices(QAudio::AudioInput); - auto outputDevices = getAvailableDevices(QAudio::AudioOutput); - + auto inputDevices = getAvailableDevices(QAudio::AudioInput, _hmdInputName); + auto outputDevices = getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); + checkDefaultChanges(inputDevices); checkDefaultChanges(outputDevices); @@ -335,8 +326,8 @@ AudioClient::AudioClient() { connect(&_receivedAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioClient::handleMismatchAudioFormat); // initialize wasapi; if getAvailableDevices is called from the CheckDevicesThread before this, it will crash - getAvailableDevices(QAudio::AudioInput); - getAvailableDevices(QAudio::AudioOutput); + getAvailableDevices(QAudio::AudioInput, _hmdInputName); + getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); // start a thread to detect any device changes _checkDevicesTimer = new QTimer(this); @@ -436,9 +427,9 @@ void AudioClient::setAudioPaused(bool pause) { } } -HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, bool isHmd=false) { +HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, const QString& hmdName, bool isHmd=false) { HifiAudioDeviceInfo result; - foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode)) { + foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode,hmdName)) { if (audioDevice.deviceName().trimmed() == deviceName.trimmed()) { if ((!isHmd && audioDevice.getDeviceType() != HifiAudioDeviceInfo::hmd) || (isHmd && audioDevice.getDeviceType() != HifiAudioDeviceInfo::desktop)) { result = audioDevice; @@ -493,7 +484,7 @@ QString AudioClient::getWinDeviceName(wchar_t* guid) { #endif -HifiAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { +HifiAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode, const QString& hmdName) { QString deviceName = defaultAudioDeviceName(mode); #if defined (Q_OS_ANDROID) if (mode == QAudio::AudioInput) { @@ -509,7 +500,7 @@ HifiAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { } } #endif - return getNamedAudioDeviceForMode(mode, deviceName); + return getNamedAudioDeviceForMode(mode, deviceName, hmdName); } QString defaultAudioDeviceName(QAudio::Mode mode) { @@ -615,7 +606,8 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { } bool AudioClient::getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName) { - return (getNamedAudioDeviceForMode(mode, deviceName).deviceName() == deviceName); + QString hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; + return (getNamedAudioDeviceForMode(mode, deviceName, hmdName).deviceName() == deviceName); } @@ -778,11 +770,11 @@ void AudioClient::start() { _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(OUTPUT_CHANNEL_COUNT); - HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput); + HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput, _hmdInputName); qCDebug(audioclient) << "The default audio input device is" << inputDeviceInfo.deviceName(); bool inputFormatSupported = switchInputToAudioDevice(inputDeviceInfo); - HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); + HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput, _hmdOutputName); qCDebug(audioclient) << "The default audio output device is" << outputDeviceInfo.deviceName(); bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo); @@ -1021,7 +1013,8 @@ bool AudioClient::switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo } bool AudioClient::switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd) { - return switchAudioDevice(mode, getNamedAudioDeviceForMode(mode, deviceName, isHmd)); + QString hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; + return switchAudioDevice(mode, getNamedAudioDeviceForMode(mode, deviceName, hmdName, isHmd)); } void AudioClient::configureReverb() { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index ce63c43cff..9628de99e8 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -313,7 +313,7 @@ private: // background tasks void checkDevices(); void checkPeakValues(); - + void setHmdAudioName(QAudio::Mode mode, const QString& name); void outputFormatChanged(); void handleAudioInput(QByteArray& audioBuffer); void prepareLocalAudioInjectors(std::unique_ptr localAudioLock = nullptr); @@ -481,6 +481,9 @@ private: QList _inputDevices; QList _outputDevices; + QString _hmdInputName { QString() }; + QString _hmdOutputName{ QString() }; + AudioFileWav _audioFileWav; bool _hasReceivedFirstPacket { false }; diff --git a/libraries/plugins/src/plugins/PluginManager.cpp b/libraries/plugins/src/plugins/PluginManager.cpp index 660159d9bd..784de6bdea 100644 --- a/libraries/plugins/src/plugins/PluginManager.cpp +++ b/libraries/plugins/src/plugins/PluginManager.cpp @@ -224,13 +224,7 @@ const OculusPlatformPluginPointer PluginManager::getOculusPlatformPlugin() { } DisplayPluginList PluginManager::getAllDisplayPlugins() { - if (thread() != QThread::currentThread()) { - DisplayPluginList list; - QMetaObject::invokeMethod(this, "getAllDisplayPlugins", Qt::BlockingQueuedConnection, Q_RETURN_ARG(DisplayPluginList, list)); - return list; - } else { - return _displayPlugins; - } + return _displayPlugins; } const DisplayPluginList& PluginManager::getDisplayPlugins() { From 9370ce9b763bbacd94449e267d2b38fe4ef259d0 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 10:24:30 -0700 Subject: [PATCH 23/33] removed items --- libraries/audio-client/CMakeLists.txt | 1 - libraries/audio-client/src/AudioClient.h | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index e1d90334ff..6b88292dd4 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -6,7 +6,6 @@ setup_hifi_library(Network Multimedia ${PLATFORM_QT_COMPONENTS}) link_hifi_libraries(audio plugins) include_hifi_library_headers(shared) include_hifi_library_headers(networking) -include_hifi_library_headers(gpu) if (ANDROID) else () diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 9628de99e8..fadc599228 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -243,6 +243,8 @@ public slots: // calling with a null QAudioDevice will use the system default bool switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo& deviceInfo = HifiAudioDeviceInfo()); bool switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd); + void setHmdAudioName(QAudio::Mode mode, const QString& name); + // Qt opensles plugin is not able to detect when the headset is plugged in void setHeadsetPluggedIn(bool pluggedIn); @@ -313,7 +315,6 @@ private: // background tasks void checkDevices(); void checkPeakValues(); - void setHmdAudioName(QAudio::Mode mode, const QString& name); void outputFormatChanged(); void handleAudioInput(QByteArray& audioBuffer); void prepareLocalAudioInjectors(std::unique_ptr localAudioLock = nullptr); From 77f5b985a1a7c0c7e9431de2c397a416307d9308 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 11:19:59 -0700 Subject: [PATCH 24/33] added read/write lock to hmd name --- libraries/audio-client/src/AudioClient.cpp | 43 +++++++++++++++++----- libraries/audio-client/src/AudioClient.h | 2 +- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index e966274a9b..c81823d14b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -83,6 +83,7 @@ Mutex _recordMutex; QString defaultAudioDeviceName(QAudio::Mode mode); void AudioClient::setHmdAudioName(QAudio::Mode mode, const QString& name) { + QWriteLocker lock(&_hmdNameLock); if (mode == QAudio::AudioInput) { _hmdInputName = name; } else { @@ -140,8 +141,17 @@ void AudioClient::checkDevices() { return; } - auto inputDevices = getAvailableDevices(QAudio::AudioInput, _hmdInputName); - auto outputDevices = getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); + QString hmdInputName; + QString hmdOutputName; + + { + QReadLocker readLock(&_hmdNameLock); + hmdInputName = _hmdInputName; + hmdOutputName = _hmdOutputName; + } + + auto inputDevices = getAvailableDevices(QAudio::AudioInput, hmdInputName); + auto outputDevices = getAvailableDevices(QAudio::AudioOutput, hmdOutputName); checkDefaultChanges(inputDevices); checkDefaultChanges(outputDevices); @@ -325,10 +335,12 @@ AudioClient::AudioClient() { connect(&_receivedAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioClient::handleMismatchAudioFormat); - // initialize wasapi; if getAvailableDevices is called from the CheckDevicesThread before this, it will crash - getAvailableDevices(QAudio::AudioInput, _hmdInputName); - getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); - + { + QReadLocker readLock(&_hmdNameLock); + // initialize wasapi; if getAvailableDevices is called from the CheckDevicesThread before this, it will crash + getAvailableDevices(QAudio::AudioInput, _hmdInputName); + getAvailableDevices(QAudio::AudioOutput, _hmdOutputName); + } // start a thread to detect any device changes _checkDevicesTimer = new QTimer(this); const unsigned long DEVICE_CHECK_INTERVAL_MSECS = 2 * 1000; @@ -606,6 +618,7 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { } bool AudioClient::getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName) { + QReadLocker readLock(&_hmdNameLock); QString hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; return (getNamedAudioDeviceForMode(mode, deviceName, hmdName).deviceName() == deviceName); } @@ -769,12 +782,20 @@ void AudioClient::start() { _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(OUTPUT_CHANNEL_COUNT); + + QString inputName; + QString outputName; + { + QReadLocker readLock(&_hmdNameLock); + inputName = _hmdInputName; + outputName = _hmdOutputName; + } - HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput, _hmdInputName); + HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput, inputName); qCDebug(audioclient) << "The default audio input device is" << inputDeviceInfo.deviceName(); bool inputFormatSupported = switchInputToAudioDevice(inputDeviceInfo); - HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput, _hmdOutputName); + HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput, outputName); qCDebug(audioclient) << "The default audio output device is" << outputDeviceInfo.deviceName(); bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo); @@ -1013,7 +1034,11 @@ bool AudioClient::switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo } bool AudioClient::switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd) { - QString hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; + QString hmdName; + { + QReadLocker readLock(&_hmdNameLock); + hmdName = mode == QAudio::AudioInput ? _hmdInputName : _hmdOutputName; + } return switchAudioDevice(mode, getNamedAudioDeviceForMode(mode, deviceName, hmdName, isHmd)); } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index fadc599228..4e6abd3509 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -508,7 +508,7 @@ private: #endif AudioSolo _solo; - + QReadWriteLock _hmdNameLock; Mutex _checkDevicesMutex; QTimer* _checkDevicesTimer { nullptr }; Mutex _checkPeakValuesMutex; From 7d5dcbfc77e3cf047504d48f428994689745e9ce Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 11:58:04 -0700 Subject: [PATCH 25/33] space cleanup --- interface/src/scripting/AudioDevices.cpp | 1 - libraries/audio-client/src/AudioClient.cpp | 2 +- libraries/audio-client/src/AudioClient.h | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index 8737cd1296..8a4e0925e3 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -69,7 +69,6 @@ static QString getTargetDevice(bool hmd, QAudio::Mode mode) { return deviceName; } - static QString getHmdAudioDeviceName(QAudio::Mode mode) { foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getAllDisplayPlugins()) { if (displayPlugin && displayPlugin->isHmd()) { diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index c81823d14b..4b7d642a71 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -116,6 +116,7 @@ QList getAvailableDevices(QAudio::Mode mode, const QString& defaultDesktopDevice = HifiAudioDeviceInfo(devices.first(), true, mode, HifiAudioDeviceInfo::desktop); } newDevices.push_front(defaultDesktopDevice); + if (!hmdName.isNull()) { HifiAudioDeviceInfo hmdDevice; foreach(auto device, newDevices) { @@ -143,7 +144,6 @@ void AudioClient::checkDevices() { QString hmdInputName; QString hmdOutputName; - { QReadLocker readLock(&_hmdNameLock); hmdInputName = _hmdInputName; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 4e6abd3509..dafa81081c 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -244,7 +244,6 @@ public slots: bool switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo& deviceInfo = HifiAudioDeviceInfo()); bool switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd); void setHmdAudioName(QAudio::Mode mode, const QString& name); - // Qt opensles plugin is not able to detect when the headset is plugged in void setHeadsetPluggedIn(bool pluggedIn); @@ -315,6 +314,7 @@ private: // background tasks void checkDevices(); void checkPeakValues(); + void outputFormatChanged(); void handleAudioInput(QByteArray& audioBuffer); void prepareLocalAudioInjectors(std::unique_ptr localAudioLock = nullptr); @@ -508,6 +508,7 @@ private: #endif AudioSolo _solo; + QReadWriteLock _hmdNameLock; Mutex _checkDevicesMutex; QTimer* _checkDevicesTimer { nullptr }; From ddfbb55e19fedae98d1aeba7d4d9a83299f8b025 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Tue, 29 Oct 2019 15:36:43 -0700 Subject: [PATCH 26/33] fixed old ui filtering of audio devices --- interface/resources/qml/hifi/audio/Audio.qml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/interface/resources/qml/hifi/audio/Audio.qml b/interface/resources/qml/hifi/audio/Audio.qml index fccba12a8a..eef339b854 100644 --- a/interface/resources/qml/hifi/audio/Audio.qml +++ b/interface/resources/qml/hifi/audio/Audio.qml @@ -375,14 +375,14 @@ Rectangle { x: margins.paddings interactive: false; height: contentHeight; - spacing: 4; + clip: true; model: AudioScriptingInterface.devices.input; delegate: Item { width: rightMostInputLevelPos - margins.paddings*2 - height: margins.sizeCheckBox > checkBoxInput.implicitHeight ? - margins.sizeCheckBox : checkBoxInput.implicitHeight - + height: ((type != "hmd" && bar.currentIndex === 0) || (type != "desktop" && bar.currentIndex === 1)) ? + (margins.sizeCheckBox > checkBoxInput.implicitHeight ? margins.sizeCheckBox + 4 : checkBoxInput.implicitHeight + 4) : 0 + visible: (type != "hmd" && bar.currentIndex === 0) || (type != "desktop" && bar.currentIndex === 1) AudioControls.CheckBox { id: checkBoxInput anchors.left: parent.left @@ -470,13 +470,13 @@ Rectangle { height: contentHeight; anchors.top: outputDeviceHeader.bottom; anchors.topMargin: 10; - spacing: 4; clip: true; model: AudioScriptingInterface.devices.output; delegate: Item { width: rightMostInputLevelPos - height: margins.sizeCheckBox > checkBoxOutput.implicitHeight ? - margins.sizeCheckBox : checkBoxOutput.implicitHeight + height: ((type != "hmd" && bar.currentIndex === 0) || (type != "desktop" && bar.currentIndex === 1)) ? + (margins.sizeCheckBox > checkBoxOutput.implicitHeight ? margins.sizeCheckBox + 4 : checkBoxOutput.implicitHeight + 4) : 0 + visible: (type != "hmd" && bar.currentIndex === 0) || (type != "desktop" && bar.currentIndex === 1) AudioControls.CheckBox { id: checkBoxOutput From 7953e3889f4f04b458a09d0a899730f158b605a3 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Tue, 10 Sep 2019 11:48:05 +0300 Subject: [PATCH 27/33] Wait for the avatar skeleton to finish loading before adding the avatar entities --- interface/src/AvatarBookmarks.cpp | 42 ++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/interface/src/AvatarBookmarks.cpp b/interface/src/AvatarBookmarks.cpp index d9ac522dc8..2ebe769bec 100644 --- a/interface/src/AvatarBookmarks.cpp +++ b/interface/src/AvatarBookmarks.cpp @@ -212,22 +212,36 @@ void AvatarBookmarks::loadBookmark(const QString& bookmarkName) { auto myAvatar = DependencyManager::get()->getMyAvatar(); auto treeRenderer = DependencyManager::get(); EntityTreePointer entityTree = treeRenderer ? treeRenderer->getTree() : nullptr; - myAvatar->clearWornAvatarEntities(); + + // Once the skeleton URL has been loaded, add the Avatar Entities. + // We have to wait, because otherwise the avatar entities will try to get attached to the joints + // of the *current* avatar at first. But the current avatar might have a different joints scheme + // from the new avatar, and that would cause the entities to be attached to the wrong joints. + + std::shared_ptr connection1 = std::make_shared(); + *connection1 = connect(myAvatar.get(), &MyAvatar::onLoadComplete, [this, bookmark, bookmarkName, myAvatar, connection1]() { + qCDebug(interfaceapp) << "Finish loading avatar bookmark" << bookmarkName; + QObject::disconnect(*connection1); + myAvatar->clearWornAvatarEntities(); + const float& qScale = bookmark.value(ENTRY_AVATAR_SCALE, 1.0f).toFloat(); + myAvatar->setAvatarScale(qScale); + QList attachments = bookmark.value(ENTRY_AVATAR_ATTACHMENTS, QList()).toList(); + myAvatar->setAttachmentsVariant(attachments); + QVariantList avatarEntities = bookmark.value(ENTRY_AVATAR_ENTITIES, QVariantList()).toList(); + addAvatarEntities(avatarEntities); + emit bookmarkLoaded(bookmarkName); + }); + + std::shared_ptr connection2 = std::make_shared(); + *connection2 = connect(myAvatar.get(), &MyAvatar::onLoadFailed, [this, bookmarkName, connection2]() { + qCDebug(interfaceapp) << "Failed to load avatar bookmark" << bookmarkName; + QObject::disconnect(*connection2); + }); + + qCDebug(interfaceapp) << "Start loading avatar bookmark" << bookmarkName; + const QString& avatarUrl = bookmark.value(ENTRY_AVATAR_URL, "").toString(); myAvatar->useFullAvatarURL(avatarUrl); - qCDebug(interfaceapp) << "Avatar On"; - const QList& attachments = bookmark.value(ENTRY_AVATAR_ATTACHMENTS, QList()).toList(); - - qCDebug(interfaceapp) << "Attach " << attachments; - myAvatar->setAttachmentsVariant(attachments); - - const float& qScale = bookmark.value(ENTRY_AVATAR_SCALE, 1.0f).toFloat(); - myAvatar->setAvatarScale(qScale); - - const QVariantList& avatarEntities = bookmark.value(ENTRY_AVATAR_ENTITIES, QVariantList()).toList(); - addAvatarEntities(avatarEntities); - - emit bookmarkLoaded(bookmarkName); } } } From 1a42ad0736cb6d596a88db41091c8f51f3e434ee Mon Sep 17 00:00:00 2001 From: dante ruiz Date: Wed, 30 Oct 2019 09:27:37 -0700 Subject: [PATCH 28/33] update launcher text --- .../qml/HFBase/CreateAccountBase.qml | 22 ++++--------------- launchers/qt/resources/qml/HFBase/Error.qml | 2 +- .../qt/resources/qml/HFBase/LoginBase.qml | 2 +- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/launchers/qt/resources/qml/HFBase/CreateAccountBase.qml b/launchers/qt/resources/qml/HFBase/CreateAccountBase.qml index c419878413..f788eeaa4d 100644 --- a/launchers/qt/resources/qml/HFBase/CreateAccountBase.qml +++ b/launchers/qt/resources/qml/HFBase/CreateAccountBase.qml @@ -8,7 +8,7 @@ import HQLauncher 1.0 Item { id: root anchors.centerIn: parent - property string titleText: "Sign in and pick a password" + property string titleText: "Create Your Username and Password" property string usernamePlaceholder: "Username" property string passwordPlaceholder: "Set a password (must be at least 6 characters)" property int marginLeft: root.width * 0.15 @@ -28,6 +28,7 @@ Item { HFTextHeader { id: title width: 481 + wrapMode: Text.WordWrap lineHeight: 35 lineHeightMode: Text.FixedHeight text: LauncherState.lastSignupErrorMessage.length == 0 ? root.titleText : "Uh oh" @@ -39,21 +40,6 @@ Item { } } - HFTextRegular { - id: instruction - width: 425 - - text: "Use the email address you applied for access with" - visible: LauncherState.lastSignupErrorMessage.length == 0 - - anchors { - left: root.left - leftMargin: root.marginLeft - top: title.bottom - topMargin: 18 - } - } - HFTextError { id: error @@ -88,10 +74,10 @@ Item { enabled: root.enabled - placeholderText: "Email Address" + placeholderText: "Verify Your Email" seperatorColor: Qt.rgba(1, 1, 1, 0.3) anchors { - top: instruction.bottom + top: error.visible ? error.bottom : title.bottom left: root.left leftMargin: root.marginLeft topMargin: 18 diff --git a/launchers/qt/resources/qml/HFBase/Error.qml b/launchers/qt/resources/qml/HFBase/Error.qml index 98eb1d17ac..5f93931ced 100644 --- a/launchers/qt/resources/qml/HFBase/Error.qml +++ b/launchers/qt/resources/qml/HFBase/Error.qml @@ -45,7 +45,7 @@ Item { HFTextRegular { id: description - text: "We seem to have a problem.\n Please restart Launcher." + text: "We seem to have a problem.\n Please restart." anchors { top: header.bottom diff --git a/launchers/qt/resources/qml/HFBase/LoginBase.qml b/launchers/qt/resources/qml/HFBase/LoginBase.qml index 1df9950dd4..1f526edf19 100644 --- a/launchers/qt/resources/qml/HFBase/LoginBase.qml +++ b/launchers/qt/resources/qml/HFBase/LoginBase.qml @@ -77,7 +77,7 @@ Item { width: 430 text: LauncherState.lastUsedUsername - placeholderText: "Username" + placeholderText: "Username or Email address" seperatorColor: Qt.rgba(1, 1, 1, 0.3) anchors { From d48097ce2c81a52d87e25fd2b566f11f78cf1d02 Mon Sep 17 00:00:00 2001 From: dante ruiz Date: Wed, 30 Oct 2019 10:41:17 -0700 Subject: [PATCH 29/33] Make requested changes --- launchers/qt/resources/qml/HFBase/Error.qml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/launchers/qt/resources/qml/HFBase/Error.qml b/launchers/qt/resources/qml/HFBase/Error.qml index 5f93931ced..8e45c6accb 100644 --- a/launchers/qt/resources/qml/HFBase/Error.qml +++ b/launchers/qt/resources/qml/HFBase/Error.qml @@ -45,7 +45,7 @@ Item { HFTextRegular { id: description - text: "We seem to have a problem.\n Please restart." + text: "We seem to have a problem." anchors { top: header.bottom @@ -54,6 +54,16 @@ Item { } } + HFTextRegular { + text: "Please restart." + + anchors { + top: description.bottom + topMargin: 1 + horizontalCenter: header.horizontalCenter + } + } + HFButton { id: button From 39e10926606ef1768c9679f92f4423d4918b1f1f Mon Sep 17 00:00:00 2001 From: amer cerkic Date: Wed, 30 Oct 2019 11:34:41 -0700 Subject: [PATCH 30/33] working on hmd fix --- interface/src/scripting/AudioDevices.cpp | 28 +++++++++++------------- interface/src/scripting/AudioDevices.h | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index 8a4e0925e3..306d822ed4 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -271,24 +271,22 @@ std::shared_ptr getSimilarDevice(const QString& deviceNa return devices[minDistanceIndex]; } -void AudioDeviceList::onDevicesChanged(const QList& devices) { + +void AudioDeviceList::onDevicesChanged(QAudio::Mode mode, const QList& devices) { beginResetModel(); QList> newDevices; bool hmdIsSelected = false; bool desktopIsSelected = false; - //getting hmd mode - if (devices.size() > 0) { - auto mode = devices.first().getMode(); - QString name = getHmdAudioDeviceName(mode); - if (!name.isEmpty()) { - auto client = DependencyManager::get().data(); - QMetaObject::invokeMethod(client, "setHmdAudioName", - Q_ARG(QAudio::Mode, mode), - Q_ARG(const QString&, name)); - } + QString name = getHmdAudioDeviceName(mode); + if (!name.isEmpty()) { + auto client = DependencyManager::get().data(); + QMetaObject::invokeMethod(client, "setHmdAudioName", + Q_ARG(QAudio::Mode, mode), + Q_ARG(const QString&, name)); } + if (!_backupSelectedDesktopDeviceName.isEmpty() && !_backupSelectedHMDDeviceName.isEmpty()) { foreach(const HifiAudioDeviceInfo& deviceInfo, devices) { @@ -452,8 +450,8 @@ AudioDevices::AudioDevices(bool& contextIsHMD) : _contextIsHMD(contextIsHMD) { const QList& devicesOutput = client->getAudioDevices(QAudio::AudioOutput); //setup devices - _inputs.onDevicesChanged(devicesInput); - _outputs.onDevicesChanged(devicesOutput); + _inputs.onDevicesChanged(QAudio::AudioInput, devicesInput); + _outputs.onDevicesChanged(QAudio::AudioOutput,devicesOutput); } AudioDevices::~AudioDevices() {} @@ -552,14 +550,14 @@ void AudioDevices::onDevicesChanged(QAudio::Mode mode, const QList& devices); + void onDevicesChanged(QAudio::Mode mode, const QList& devices); protected: friend class AudioDevices; From 4b4b722875cd64aaa786d637aa5cb592586b0600 Mon Sep 17 00:00:00 2001 From: amer cerkic Date: Wed, 30 Oct 2019 16:27:19 -0700 Subject: [PATCH 31/33] fixed issues related to context switching and selection with hmd --- interface/src/scripting/AudioDevices.cpp | 36 ++++++++++++---------- libraries/audio-client/src/AudioClient.cpp | 18 +---------- 2 files changed, 21 insertions(+), 33 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index 306d822ed4..d66cb84dfe 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -69,18 +69,25 @@ static QString getTargetDevice(bool hmd, QAudio::Mode mode) { return deviceName; } -static QString getHmdAudioDeviceName(QAudio::Mode mode) { +static void checkHmdDefaultsChange(QAudio::Mode mode) { + QString name; foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getAllDisplayPlugins()) { if (displayPlugin && displayPlugin->isHmd()) { if (mode == QAudio::AudioInput) { - return displayPlugin->getPreferredAudioInDevice(); + name = displayPlugin->getPreferredAudioInDevice(); } else { - return displayPlugin->getPreferredAudioOutDevice(); + name = displayPlugin->getPreferredAudioOutDevice(); } break; } } - return QString(); + + if (!name.isEmpty()) { + auto client = DependencyManager::get().data(); + QMetaObject::invokeMethod(client, "setHmdAudioName", + Q_ARG(QAudio::Mode, mode), + Q_ARG(const QString&, name)); + } } Qt::ItemFlags AudioDeviceList::_flags { Qt::ItemIsSelectable | Qt::ItemIsEnabled }; @@ -278,16 +285,8 @@ void AudioDeviceList::onDevicesChanged(QAudio::Mode mode, const QList> newDevices; bool hmdIsSelected = false; bool desktopIsSelected = false; - - QString name = getHmdAudioDeviceName(mode); - if (!name.isEmpty()) { - auto client = DependencyManager::get().data(); - QMetaObject::invokeMethod(client, "setHmdAudioName", - Q_ARG(QAudio::Mode, mode), - Q_ARG(const QString&, name)); - } - + checkHmdDefaultsChange(mode); if (!_backupSelectedDesktopDeviceName.isEmpty() && !_backupSelectedHMDDeviceName.isEmpty()) { foreach(const HifiAudioDeviceInfo& deviceInfo, devices) { for (bool isHMD : {false, true}) { @@ -441,6 +440,9 @@ AudioDevices::AudioDevices(bool& contextIsHMD) : _contextIsHMD(contextIsHMD) { connect(client, &AudioClient::deviceChanged, this, &AudioDevices::onDeviceChanged, Qt::QueuedConnection); connect(client, &AudioClient::devicesChanged, this, &AudioDevices::onDevicesChanged, Qt::QueuedConnection); connect(client, &AudioClient::peakValueListChanged, &_inputs, &AudioInputDeviceList::onPeakValueListChanged, Qt::QueuedConnection); + + checkHmdDefaultsChange(QAudio::AudioInput); + checkHmdDefaultsChange(QAudio::AudioOutput); _inputs.onDeviceChanged(client->getActiveAudioDevice(QAudio::AudioInput), contextIsHMD); _outputs.onDeviceChanged(client->getActiveAudioDevice(QAudio::AudioOutput), contextIsHMD); @@ -449,9 +451,11 @@ AudioDevices::AudioDevices(bool& contextIsHMD) : _contextIsHMD(contextIsHMD) { const QList& devicesInput = client->getAudioDevices(QAudio::AudioInput); const QList& devicesOutput = client->getAudioDevices(QAudio::AudioOutput); - //setup devices - _inputs.onDevicesChanged(QAudio::AudioInput, devicesInput); - _outputs.onDevicesChanged(QAudio::AudioOutput,devicesOutput); + if (devicesInput.size() > 0 && devicesOutput.size() > 0) { + //setup devices + _inputs.onDevicesChanged(QAudio::AudioInput, devicesInput); + _outputs.onDevicesChanged(QAudio::AudioOutput, devicesOutput); + } } AudioDevices::~AudioDevices() {} diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 4b7d642a71..d8741e4aa7 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -791,23 +791,7 @@ void AudioClient::start() { outputName = _hmdOutputName; } - HifiAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput, inputName); - qCDebug(audioclient) << "The default audio input device is" << inputDeviceInfo.deviceName(); - bool inputFormatSupported = switchInputToAudioDevice(inputDeviceInfo); - - HifiAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput, outputName); - qCDebug(audioclient) << "The default audio output device is" << outputDeviceInfo.deviceName(); - bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo); - - if (!inputFormatSupported) { - qCDebug(audioclient) << "Unable to set up audio input because of a problem with input format."; - qCDebug(audioclient) << "The closest format available is" << inputDeviceInfo.getDevice().nearestFormat(_desiredInputFormat); - } - - if (!outputFormatSupported) { - qCDebug(audioclient) << "Unable to set up audio output because of a problem with output format."; - qCDebug(audioclient) << "The closest format available is" << outputDeviceInfo.getDevice().nearestFormat(_desiredOutputFormat); - } + #if defined(Q_OS_ANDROID) connect(&_checkInputTimer, &QTimer::timeout, this, &AudioClient::checkInputTimeout); _checkInputTimer.start(CHECK_INPUT_READS_MSECS); From 52e0f54d0c76456b7296bf10a321ba7c65817df2 Mon Sep 17 00:00:00 2001 From: amerhifi Date: Thu, 31 Oct 2019 07:21:16 -0700 Subject: [PATCH 32/33] addressing comments --- interface/src/scripting/AudioDevices.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index d66cb84dfe..688b4df8cd 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -554,14 +554,14 @@ void AudioDevices::onDevicesChanged(QAudio::Mode mode, const QList Date: Thu, 31 Oct 2019 11:29:46 -0700 Subject: [PATCH 33/33] Attempt to fix DEV-2615 and DEV-2618 with more menu scripting interface guards --- .../src/scripting/MenuScriptingInterface.cpp | 119 +++++++++++++++--- 1 file changed, 100 insertions(+), 19 deletions(-) diff --git a/interface/src/scripting/MenuScriptingInterface.cpp b/interface/src/scripting/MenuScriptingInterface.cpp index 1020c12733..9e7f6fdc2b 100644 --- a/interface/src/scripting/MenuScriptingInterface.cpp +++ b/interface/src/scripting/MenuScriptingInterface.cpp @@ -32,106 +32,187 @@ void MenuScriptingInterface::menuItemTriggered() { } void MenuScriptingInterface::addMenu(const QString& menu, const QString& grouping) { - QMetaObject::invokeMethod(Menu::getInstance(), "addMenu", Q_ARG(const QString&, menu), Q_ARG(const QString&, grouping)); + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "addMenu", Q_ARG(const QString&, menu), Q_ARG(const QString&, grouping)); } void MenuScriptingInterface::removeMenu(const QString& menu) { - QMetaObject::invokeMethod(Menu::getInstance(), "removeMenu", Q_ARG(const QString&, menu)); + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "removeMenu", Q_ARG(const QString&, menu)); } bool MenuScriptingInterface::menuExists(const QString& menu) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return false; + } + if (QThread::currentThread() == qApp->thread()) { - Menu* menuInstance = Menu::getInstance(); return menuInstance && menuInstance->menuExists(menu); } + bool result { false }; - BLOCKING_INVOKE_METHOD(Menu::getInstance(), "menuExists", + + BLOCKING_INVOKE_METHOD(menuInstance, "menuExists", Q_RETURN_ARG(bool, result), Q_ARG(const QString&, menu)); + return result; } void MenuScriptingInterface::addSeparator(const QString& menuName, const QString& separatorName) { - QMetaObject::invokeMethod(Menu::getInstance(), "addSeparator", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "addSeparator", Q_ARG(const QString&, menuName), Q_ARG(const QString&, separatorName)); } void MenuScriptingInterface::removeSeparator(const QString& menuName, const QString& separatorName) { - QMetaObject::invokeMethod(Menu::getInstance(), "removeSeparator", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "removeSeparator", Q_ARG(const QString&, menuName), Q_ARG(const QString&, separatorName)); } void MenuScriptingInterface::addMenuItem(const MenuItemProperties& properties) { - QMetaObject::invokeMethod(Menu::getInstance(), "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); } void MenuScriptingInterface::addMenuItem(const QString& menu, const QString& menuitem, const QString& shortcutKey) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + MenuItemProperties properties(menu, menuitem, shortcutKey); - QMetaObject::invokeMethod(Menu::getInstance(), "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); + QMetaObject::invokeMethod(menuInstance, "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); } void MenuScriptingInterface::addMenuItem(const QString& menu, const QString& menuitem) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + MenuItemProperties properties(menu, menuitem); - QMetaObject::invokeMethod(Menu::getInstance(), "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); + QMetaObject::invokeMethod(menuInstance, "addMenuItem", Q_ARG(const MenuItemProperties&, properties)); } void MenuScriptingInterface::removeMenuItem(const QString& menu, const QString& menuitem) { - QMetaObject::invokeMethod(Menu::getInstance(), "removeMenuItem", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + QMetaObject::invokeMethod(menuInstance, "removeMenuItem", Q_ARG(const QString&, menu), Q_ARG(const QString&, menuitem)); }; bool MenuScriptingInterface::menuItemExists(const QString& menu, const QString& menuitem) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return false; + } + if (QThread::currentThread() == qApp->thread()) { - Menu* menuInstance = Menu::getInstance(); return menuInstance && menuInstance->menuItemExists(menu, menuitem); } + bool result { false }; - BLOCKING_INVOKE_METHOD(Menu::getInstance(), "menuItemExists", + + BLOCKING_INVOKE_METHOD(menuInstance, "menuItemExists", Q_RETURN_ARG(bool, result), Q_ARG(const QString&, menu), Q_ARG(const QString&, menuitem)); + return result; } bool MenuScriptingInterface::isOptionChecked(const QString& menuOption) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return false; + } + if (QThread::currentThread() == qApp->thread()) { - Menu* menuInstance = Menu::getInstance(); return menuInstance && menuInstance->isOptionChecked(menuOption); } + bool result { false }; - BLOCKING_INVOKE_METHOD(Menu::getInstance(), "isOptionChecked", + + BLOCKING_INVOKE_METHOD(menuInstance, "isOptionChecked", Q_RETURN_ARG(bool, result), Q_ARG(const QString&, menuOption)); return result; } void MenuScriptingInterface::setIsOptionChecked(const QString& menuOption, bool isChecked) { - QMetaObject::invokeMethod(Menu::getInstance(), "setIsOptionChecked", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "setIsOptionChecked", Q_ARG(const QString&, menuOption), Q_ARG(bool, isChecked)); } bool MenuScriptingInterface::isMenuEnabled(const QString& menuOption) { + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return false; + } + if (QThread::currentThread() == qApp->thread()) { - Menu* menuInstance = Menu::getInstance(); return menuInstance && menuInstance->isMenuEnabled(menuOption); } + bool result { false }; - BLOCKING_INVOKE_METHOD(Menu::getInstance(), "isMenuEnabled", + + BLOCKING_INVOKE_METHOD(menuInstance, "isMenuEnabled", Q_RETURN_ARG(bool, result), Q_ARG(const QString&, menuOption)); + return result; } void MenuScriptingInterface::setMenuEnabled(const QString& menuOption, bool isChecked) { - QMetaObject::invokeMethod(Menu::getInstance(), "setMenuEnabled", + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "setMenuEnabled", Q_ARG(const QString&, menuOption), Q_ARG(bool, isChecked)); } void MenuScriptingInterface::triggerOption(const QString& menuOption) { - QMetaObject::invokeMethod(Menu::getInstance(), "triggerOption", Q_ARG(const QString&, menuOption)); + Menu* menuInstance = Menu::getInstance(); + if (!menuInstance) { + return; + } + + QMetaObject::invokeMethod(menuInstance, "triggerOption", Q_ARG(const QString&, menuOption)); }