Merge pull request #3204 from Barnold1953/jsBlendShapes

Exposed Avatar Blendshape control to JS

assignment-client/src/Agent.cpp

@@ -229,7 +229,8 @@ void Agent::run() {
     
     // setup an Avatar for the script to use
     AvatarData scriptedAvatar;
-    
+    scriptedAvatar.setForceFaceshiftConnected(true);
+
     // call model URL setters with empty URLs so our avatar, if user, will have the default models
     scriptedAvatar.setFaceModelURL(QUrl());
     scriptedAvatar.setSkeletonModelURL(QUrl());

examples/bot_randomExpression.js

@@ -0,0 +1,144 @@
+//
+//  bot_randomExpression.js
+//  examples
+//
+//  Created by Ben Arnold on 7/23/14.
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  This is an example script that demonstrates an NPC avatar with
+//  random facial expressions.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+function getRandomFloat(min, max) {
+    return Math.random() * (max - min) + min;
+}
+
+function getRandomInt (min, max) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function printVector(string, vector) {
+    print(string + " " + vector.x + ", " + vector.y + ", " + vector.z);
+}
+
+var timePassed = 0.0;
+var updateSpeed = 3.0;
+
+var X_MIN = 5.0;
+var X_MAX = 15.0;
+var Z_MIN = 5.0;
+var Z_MAX = 15.0;
+var Y_PELVIS = 1.0;
+
+// pick an integer between 1 and 100 for the body model for this bot
+botNumber = getRandomInt(1, 100);
+
+newFaceFilePrefix = "ron";
+
+newBodyFilePrefix = "bot" + botNumber;
+
+// set the face model fst using the bot number
+// there is no need to change the body model - we're using the default
+Avatar.faceModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/" + newFaceFilePrefix + ".fst";
+Avatar.skeletonModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/" + newBodyFilePrefix + ".fst";
+Avatar.billboardURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/billboards/bot" + botNumber + ".png";
+
+Agent.isAvatar = true;
+Agent.isListeningToAudioStream = true;
+
+// change the avatar's position to the random one
+Avatar.position = { x: getRandomFloat(X_MIN, X_MAX), y: Y_PELVIS, z: getRandomFloat(Z_MIN, Z_MAX) };;  
+printVector("New bot, position = ", Avatar.position);
+
+var allBlendShapes = [];
+var targetBlendCoefficient = [];
+var currentBlendCoefficient = [];
+
+function addBlendShape(s) {
+    allBlendShapes[allBlendShapes.length] = s;
+}
+
+//It is imperative that the following blendshapes are all present and are in the correct order
+addBlendShape("EyeBlink_L");
+addBlendShape("EyeBlink_R");
+addBlendShape("EyeSquint_L");
+addBlendShape("EyeSquint_R");
+addBlendShape("EyeDown_L");
+addBlendShape("EyeDown_R");
+addBlendShape("EyeIn_L");
+addBlendShape("EyeIn_R");
+addBlendShape("EyeOpen_L");
+addBlendShape("EyeOpen_R");
+addBlendShape("EyeOut_L");
+addBlendShape("EyeOut_R");
+addBlendShape("EyeUp_L");
+addBlendShape("EyeUp_R");
+addBlendShape("BrowsD_L");
+addBlendShape("BrowsD_R");
+addBlendShape("BrowsU_C");
+addBlendShape("BrowsU_L");
+addBlendShape("BrowsU_R");
+addBlendShape("JawFwd");
+addBlendShape("JawLeft");
+addBlendShape("JawOpen");
+addBlendShape("JawChew");
+addBlendShape("JawRight");
+addBlendShape("MouthLeft");
+addBlendShape("MouthRight");
+addBlendShape("MouthFrown_L");
+addBlendShape("MouthFrown_R");
+addBlendShape("MouthSmile_L");
+addBlendShape("MouthSmile_R");
+addBlendShape("MouthDimple_L");
+addBlendShape("MouthDimple_R");
+addBlendShape("LipsStretch_L");
+addBlendShape("LipsStretch_R");
+addBlendShape("LipsUpperClose");
+addBlendShape("LipsLowerClose");
+addBlendShape("LipsUpperUp");
+addBlendShape("LipsLowerDown");
+addBlendShape("LipsUpperOpen");
+addBlendShape("LipsLowerOpen");
+addBlendShape("LipsFunnel");
+addBlendShape("LipsPucker");
+addBlendShape("ChinLowerRaise");
+addBlendShape("ChinUpperRaise");
+addBlendShape("Sneer");
+addBlendShape("Puff");
+addBlendShape("CheekSquint_L");
+addBlendShape("CheekSquint_R");
+
+for (var i = 0; i < allBlendShapes.length; i++) {
+    targetBlendCoefficient[i] = 0;
+    currentBlendCoefficient[i] = 0;
+}
+
+function setRandomExpression() {
+    for (var i = 0; i < allBlendShapes.length; i++) {
+        targetBlendCoefficient[i] = Math.random();
+    }
+}
+
+var expressionChangeSpeed = 0.1;
+
+function updateBlendShapes(deltaTime) {
+  
+    for (var i = 0; i < allBlendShapes.length; i++) {
+        currentBlendCoefficient[i] += (targetBlendCoefficient[i] - currentBlendCoefficient[i]) * expressionChangeSpeed;
+        Avatar.setBlendshape(allBlendShapes[i], currentBlendCoefficient[i]);
+    }
+}
+
+function update(deltaTime) {
+    timePassed += deltaTime;
+    if (timePassed > updateSpeed) {
+        timePassed = 0;
+        setRandomExpression();
+    }
+    updateBlendShapes(deltaTime);
+}
+
+Script.update.connect(update);

examples/editVoxels.js

@@ -51,7 +51,6 @@ var lastVoxelScale = 0;
 var dragStart = { x: 0, y: 0 };
 var wheelPixelsMoved = 0;
 
-
 var mouseX = 0;
 var mouseY = 0;
 
@@ -168,7 +167,16 @@ var voxelPreview = Overlays.addOverlay("cube", {
                                        lineWidth: 4
                                        });
 
-var linePreviewTop = Overlays.addOverlay("line3d", {
+var linePreviewTop = [];
+var linePreviewBottom = [];
+var linePreviewLeft = [];
+var linePreviewRight = [];
+
+// Currend cursor index
+var currentCursor = 0;
+                        
+function addLineOverlay() {
+    return Overlays.addOverlay("line3d", {
                                          position: { x: 0, y: 0, z: 0},
                                          end: { x: 0, y: 0, z: 0},
                                          color: { red: 255, green: 255, blue: 255},
@@ -176,34 +184,24 @@ var linePreviewTop = Overlays.addOverlay("line3d", {
                                          visible: false,
                                          lineWidth: previewLineWidth
                                          });
+}
+                  
+//Cursor line previews for up to three cursors                  
+linePreviewTop[0] = addLineOverlay();       
+linePreviewTop[1] = addLineOverlay();
+linePreviewTop[2] = addLineOverlay();
 
-var linePreviewBottom = Overlays.addOverlay("line3d", {
+linePreviewBottom[0] = addLineOverlay();
-                                            position: { x: 0, y: 0, z: 0},
+linePreviewBottom[1] = addLineOverlay();
-                                            end: { x: 0, y: 0, z: 0},
+linePreviewBottom[2] = addLineOverlay();
-                                            color: { red: 255, green: 255, blue: 255},
+                                            
-                                            alpha: 1,
+linePreviewLeft[0] = addLineOverlay();
-                                            visible: false,
+linePreviewLeft[1] = addLineOverlay();
-                                            lineWidth: previewLineWidth
+linePreviewLeft[2] = addLineOverlay();
-                                            });
+                                          
-
+linePreviewRight[0] = addLineOverlay();
-var linePreviewLeft = Overlays.addOverlay("line3d", {
+linePreviewRight[1] = addLineOverlay();
-                                          position: { x: 0, y: 0, z: 0},
+linePreviewRight[2] = addLineOverlay();
-                                          end: { x: 0, y: 0, z: 0},
-                                          color: { red: 255, green: 255, blue: 255},
-                                          alpha: 1,
-                                          visible: false,
-                                          lineWidth: previewLineWidth
-                                          });
-
-var linePreviewRight = Overlays.addOverlay("line3d", {
-                                           position: { x: 0, y: 0, z: 0},
-                                           end: { x: 0, y: 0, z: 0},
-                                           color: { red: 255, green: 255, blue: 255},
-                                           alpha: 1,
-                                           visible: false,
-                                           lineWidth: previewLineWidth
-                                           });
-
 
 // these will be used below
 var scaleSelectorWidth = 144;
@@ -809,21 +807,21 @@ function showPreviewLines() {
         var pasteVoxel = getNewPasteVoxel(pickRay);
         
         // X axis
-        Overlays.editOverlay(linePreviewBottom, {
+        Overlays.editOverlay(linePreviewBottom[currentCursor], {
                              position: pasteVoxel.origin,
                              end: {x: pasteVoxel.origin.x + pasteVoxel.voxelSize, y: pasteVoxel.origin.y, z: pasteVoxel.origin.z },
                              visible: true
                              });
         
         // Y axis
-        Overlays.editOverlay(linePreviewRight, {
+        Overlays.editOverlay(linePreviewRight[currentCursor], {
                              position: pasteVoxel.origin,
                              end: {x: pasteVoxel.origin.x, y: pasteVoxel.origin.y + pasteVoxel.voxelSize, z: pasteVoxel.origin.z },
                              visible: true
                              });
         
         // Z axis
-        Overlays.editOverlay(linePreviewTop, {
+        Overlays.editOverlay(linePreviewTop[currentCursor], {
                              position: pasteVoxel.origin,
                              end: {x: pasteVoxel.origin.x, y: pasteVoxel.origin.y, z: pasteVoxel.origin.z - pasteVoxel.voxelSize },
                              visible: true
@@ -837,10 +835,10 @@ function showPreviewLines() {
     if (intersection.intersects) {
         resultVoxel = calculateVoxelFromIntersection(intersection,"");
         Overlays.editOverlay(voxelPreview, { visible: false });
-        Overlays.editOverlay(linePreviewTop, { position: resultVoxel.topLeft, end: resultVoxel.topRight, visible: true });
+        Overlays.editOverlay(linePreviewTop[currentCursor], { position: resultVoxel.topLeft, end: resultVoxel.topRight, visible: true });
-        Overlays.editOverlay(linePreviewBottom, { position: resultVoxel.bottomLeft, end: resultVoxel.bottomRight, visible: true });
+        Overlays.editOverlay(linePreviewBottom[currentCursor], { position: resultVoxel.bottomLeft, end: resultVoxel.bottomRight, visible: true });
-        Overlays.editOverlay(linePreviewLeft, { position: resultVoxel.topLeft, end: resultVoxel.bottomLeft, visible: true });
+        Overlays.editOverlay(linePreviewLeft[currentCursor], { position: resultVoxel.topLeft, end: resultVoxel.bottomLeft, visible: true });
-        Overlays.editOverlay(linePreviewRight, { position: resultVoxel.topRight, end: resultVoxel.bottomRight, visible: true });
+        Overlays.editOverlay(linePreviewRight[currentCursor], { position: resultVoxel.topRight, end: resultVoxel.bottomRight, visible: true });
         colors[0] = {red: intersection.voxel.red, green: intersection.voxel.green , blue: intersection.voxel.blue };
         
         if (copyScale) {
@@ -849,10 +847,10 @@ function showPreviewLines() {
         moveTools();
     } else {
         Overlays.editOverlay(voxelPreview, { visible: false });
-        Overlays.editOverlay(linePreviewTop, { visible: false });
+        Overlays.editOverlay(linePreviewTop[currentCursor], { visible: false });
-        Overlays.editOverlay(linePreviewBottom, { visible: false });
+        Overlays.editOverlay(linePreviewBottom[currentCursor], { visible: false });
-        Overlays.editOverlay(linePreviewLeft, { visible: false });
+        Overlays.editOverlay(linePreviewLeft[currentCursor], { visible: false });
-        Overlays.editOverlay(linePreviewRight, { visible: false });
+        Overlays.editOverlay(linePreviewRight[currentCursor], { visible: false });
     }
 }
 
@@ -862,20 +860,20 @@ function showPreviewGuides() {
             showPreviewVoxel();
             
             // make sure alternative is hidden
-            Overlays.editOverlay(linePreviewTop, { visible: false });
+            Overlays.editOverlay(linePreviewTop[currentCursor], { visible: false });
-            Overlays.editOverlay(linePreviewBottom, { visible: false });
+            Overlays.editOverlay(linePreviewBottom[currentCursor], { visible: false });
-            Overlays.editOverlay(linePreviewLeft, { visible: false });
+            Overlays.editOverlay(linePreviewLeft[currentCursor], { visible: false });
-            Overlays.editOverlay(linePreviewRight, { visible: false });
+            Overlays.editOverlay(linePreviewRight[currentCursor], { visible: false });
         } else {
             showPreviewLines();
         }
     } else {
         // make sure all previews are off
         Overlays.editOverlay(voxelPreview, { visible: false });
-        Overlays.editOverlay(linePreviewTop, { visible: false });
+        Overlays.editOverlay(linePreviewTop[currentCursor], { visible: false });
-        Overlays.editOverlay(linePreviewBottom, { visible: false });
+        Overlays.editOverlay(linePreviewBottom[currentCursor], { visible: false });
-        Overlays.editOverlay(linePreviewLeft, { visible: false });
+        Overlays.editOverlay(linePreviewLeft[currentCursor], { visible: false });
-        Overlays.editOverlay(linePreviewRight, { visible: false });
+        Overlays.editOverlay(linePreviewRight[currentCursor], { visible: false });
     }
 }
 
@@ -968,6 +966,14 @@ function mousePressEvent(event) {
         return;
     }
     
+    if (event.deviceID == 1500) { // Left Hydra Controller
+        currentCursor = 0;
+    } else if (event.deviceID == 1501) { // Right Hydra Controller
+        currentCursor = 1;
+    } else {
+        currentCursor = 2;
+    }
+    
     var clickedOnSomething = false;
     var clickedOverlay = Overlays.getOverlayAtPoint({x: event.x, y: event.y});
     
@@ -1220,6 +1226,7 @@ function menuItemEvent(menuItem) {
 }
 
 function mouseMoveEvent(event) {
+
     if (!editToolsOn) {
         return;
     }
@@ -1227,6 +1234,14 @@ function mouseMoveEvent(event) {
         return;
     }
     
+    if (event.deviceID == 1500) { // Left Hydra Controller
+        currentCursor = 0;
+    } else if (event.deviceID == 1501) { // Right Hydra Controller
+        currentCursor = 1;
+    } else {
+        currentCursor = 2;
+    }
+    
     // Move Import Preview
     if (isImporting) {
         var pickRay = Camera.computePickRay(event.x, event.y);
@@ -1475,10 +1490,12 @@ Controller.captureKeyEvents({ text: "-" });
 
 function scriptEnding() {
     Overlays.deleteOverlay(voxelPreview);
-    Overlays.deleteOverlay(linePreviewTop);
+    for (var i = 0; i < linePreviewTop.length; i++) {
-    Overlays.deleteOverlay(linePreviewBottom);
+        Overlays.deleteOverlay(linePreviewTop[i]);
-    Overlays.deleteOverlay(linePreviewLeft);
+        Overlays.deleteOverlay(linePreviewBottom[i]);
-    Overlays.deleteOverlay(linePreviewRight);
+        Overlays.deleteOverlay(linePreviewLeft[i]);
+        Overlays.deleteOverlay(linePreviewRight[i]);
+    }
     for (s = 0; s < numColors; s++) {
         Overlays.deleteOverlay(swatches[s]);
     }

libraries/avatars/src/AvatarData.cpp

@@ -43,6 +43,7 @@ AvatarData::AvatarData() :
     _handState(0),
     _keyState(NO_KEY_DOWN),
     _isChatCirclingEnabled(false),
+    _forceFaceshiftConnected(false),
     _hasNewJointRotations(true),
     _headData(NULL),
     _handData(NULL),
@@ -80,6 +81,9 @@ QByteArray AvatarData::toByteArray() {
     // lazily allocate memory for HeadData in case we're not an Avatar instance
     if (!_headData) {
         _headData = new HeadData(this);
+        if (_forceFaceshiftConnected) {
+            _headData->_isFaceshiftConnected = true;
+        }
     }
     
     QByteArray avatarDataByteArray;

libraries/avatars/src/AvatarData.h

@@ -206,6 +206,10 @@ public:
 
     Q_INVOKABLE virtual QStringList getJointNames() const { return _jointNames; }
 
+    Q_INVOKABLE void setBlendshape(QString name, float val) { _headData->setBlendshape(name, val); }
+
+    void setForceFaceshiftConnected(bool connected) { _forceFaceshiftConnected = connected; }
+
     // key state
     void setKeyState(KeyState s) { _keyState = s; }
     KeyState keyState() const { return _keyState; }
@@ -300,7 +304,7 @@ protected:
     std::string _chatMessage;
 
     bool _isChatCirclingEnabled;
-
+    bool _forceFaceshiftConnected;
     bool _hasNewJointRotations; // set in AvatarData, cleared in Avatar
 
     HeadData* _headData;

libraries/avatars/src/HeadData.cpp

@@ -16,6 +16,8 @@
 #include "AvatarData.h"
 #include "HeadData.h"
 
+#include "../fbx/src/FBXReader.h"
+
 HeadData::HeadData(AvatarData* owningAvatar) :
     _baseYaw(0.0f),
     _basePitch(0.0f),
@@ -52,6 +54,26 @@ void HeadData::setOrientation(const glm::quat& orientation) {
     _baseRoll = eulers.z;
 }
 
+void HeadData::setBlendshape(QString name, float val) {
+    static bool hasInitializedLookupMap = false;
+    static QMap<QString, int> blendshapeLookupMap;
+    //Lazily construct a lookup map from the blendshapes
+    if (!hasInitializedLookupMap) {
+        for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) {
+            blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i; 
+        }
+    }
+
+    //Check to see if the named blendshape exists, and then set its value if it does
+    QMap<QString, int>::iterator it = blendshapeLookupMap.find(name);
+    if (it != blendshapeLookupMap.end()) {
+        if (_blendshapeCoefficients.size() <= it.value()) {
+            _blendshapeCoefficients.resize(it.value() + 1);
+        }
+        _blendshapeCoefficients[it.value()] = val;
+    }
+}
+
 void HeadData::addYaw(float yaw) {
     setBaseYaw(_baseYaw + yaw);
 }

libraries/avatars/src/HeadData.h

@@ -54,6 +54,7 @@ public:
     float getAudioAverageLoudness() const { return _audioAverageLoudness; }
 	void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; }
 
+    void setBlendshape(QString name, float val);
     const QVector<float>& getBlendshapeCoefficients() const { return _blendshapeCoefficients; }
     
     float getPupilDilation() const { return _pupilDilation; }

libraries/fbx/src/FBXReader.cpp

@@ -577,6 +577,8 @@ const char* FACESHIFT_BLENDSHAPES[] = {
     ""
 };
 
+const int NUM_FACESHIFT_BLENDSHAPES = sizeof(FACESHIFT_BLENDSHAPES) / sizeof(char*);
+
 const char* HUMANIK_JOINTS[] = {
     "RightHand",
     "RightForeArm",

libraries/fbx/src/FBXReader.h

@@ -29,6 +29,8 @@ typedef QList<FBXNode> FBXNodeList;
 
 /// The names of the blendshapes expected by Faceshift, terminated with an empty string.
 extern const char* FACESHIFT_BLENDSHAPES[];
+/// The size of FACESHIFT_BLENDSHAPES
+extern const int NUM_FACESHIFT_BLENDSHAPES;
 
 /// The names of the joints in the Maya HumanIK rig, terminated with an empty string.
 extern const char* HUMANIK_JOINTS[];