mirror of
https://github.com/overte-org/overte.git
synced 2025-04-20 02:23:57 +02:00
Merge pull request #3204 from Barnold1953/jsBlendShapes
Exposed Avatar Blendshape control to JS
This commit is contained in:
AndrewMeadows
commit
361ec5d1ee
9 changed files with 251 additions and 54 deletions
|
|
@ -229,7 +229,8 @@ void Agent::run() {
|
|||
|
||||
// setup an Avatar for the script to use
|
||||
AvatarData scriptedAvatar;
|
||||
|
||||
scriptedAvatar.setForceFaceshiftConnected(true);
|
||||
|
||||
// call model URL setters with empty URLs so our avatar, if user, will have the default models
|
||||
scriptedAvatar.setFaceModelURL(QUrl());
|
||||
scriptedAvatar.setSkeletonModelURL(QUrl());
|
||||
|
|
|
|||
144
examples/bot_randomExpression.js
Normal file
144
examples/bot_randomExpression.js
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
//
|
||||
// bot_randomExpression.js
|
||||
// examples
|
||||
//
|
||||
// Created by Ben Arnold on 7/23/14.
|
||||
// Copyright 2014 High Fidelity, Inc.
|
||||
//
|
||||
// This is an example script that demonstrates an NPC avatar with
|
||||
// random facial expressions.
|
||||
//
|
||||
// Distributed under the Apache License, Version 2.0.
|
||||
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
//
|
||||
|
||||
function getRandomFloat(min, max) {
|
||||
return Math.random() * (max - min) + min;
|
||||
}
|
||||
|
||||
function getRandomInt (min, max) {
|
||||
return Math.floor(Math.random() * (max - min + 1)) + min;
|
||||
}
|
||||
|
||||
function printVector(string, vector) {
|
||||
print(string + " " + vector.x + ", " + vector.y + ", " + vector.z);
|
||||
}
|
||||
|
||||
var timePassed = 0.0;
|
||||
var updateSpeed = 3.0;
|
||||
|
||||
var X_MIN = 5.0;
|
||||
var X_MAX = 15.0;
|
||||
var Z_MIN = 5.0;
|
||||
var Z_MAX = 15.0;
|
||||
var Y_PELVIS = 1.0;
|
||||
|
||||
// pick an integer between 1 and 100 for the body model for this bot
|
||||
botNumber = getRandomInt(1, 100);
|
||||
|
||||
newFaceFilePrefix = "ron";
|
||||
|
||||
newBodyFilePrefix = "bot" + botNumber;
|
||||
|
||||
// set the face model fst using the bot number
|
||||
// there is no need to change the body model - we're using the default
|
||||
Avatar.faceModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/" + newFaceFilePrefix + ".fst";
|
||||
Avatar.skeletonModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/" + newBodyFilePrefix + ".fst";
|
||||
Avatar.billboardURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/billboards/bot" + botNumber + ".png";
|
||||
|
||||
Agent.isAvatar = true;
|
||||
Agent.isListeningToAudioStream = true;
|
||||
|
||||
// change the avatar's position to the random one
|
||||
Avatar.position = { x: getRandomFloat(X_MIN, X_MAX), y: Y_PELVIS, z: getRandomFloat(Z_MIN, Z_MAX) };;
|
||||
printVector("New bot, position = ", Avatar.position);
|
||||
|
||||
var allBlendShapes = [];
|
||||
var targetBlendCoefficient = [];
|
||||
var currentBlendCoefficient = [];
|
||||
|
||||
function addBlendShape(s) {
|
||||
allBlendShapes[allBlendShapes.length] = s;
|
||||
}
|
||||
|
||||
//It is imperative that the following blendshapes are all present and are in the correct order
|
||||
addBlendShape("EyeBlink_L");
|
||||
addBlendShape("EyeBlink_R");
|
||||
addBlendShape("EyeSquint_L");
|
||||
addBlendShape("EyeSquint_R");
|
||||
addBlendShape("EyeDown_L");
|
||||
addBlendShape("EyeDown_R");
|
||||
addBlendShape("EyeIn_L");
|
||||
addBlendShape("EyeIn_R");
|
||||
addBlendShape("EyeOpen_L");
|
||||
addBlendShape("EyeOpen_R");
|
||||
addBlendShape("EyeOut_L");
|
||||
addBlendShape("EyeOut_R");
|
||||
addBlendShape("EyeUp_L");
|
||||
addBlendShape("EyeUp_R");
|
||||
addBlendShape("BrowsD_L");
|
||||
addBlendShape("BrowsD_R");
|
||||
addBlendShape("BrowsU_C");
|
||||
addBlendShape("BrowsU_L");
|
||||
addBlendShape("BrowsU_R");
|
||||
addBlendShape("JawFwd");
|
||||
addBlendShape("JawLeft");
|
||||
addBlendShape("JawOpen");
|
||||
addBlendShape("JawChew");
|
||||
addBlendShape("JawRight");
|
||||
addBlendShape("MouthLeft");
|
||||
addBlendShape("MouthRight");
|
||||
addBlendShape("MouthFrown_L");
|
||||
addBlendShape("MouthFrown_R");
|
||||
addBlendShape("MouthSmile_L");
|
||||
addBlendShape("MouthSmile_R");
|
||||
addBlendShape("MouthDimple_L");
|
||||
addBlendShape("MouthDimple_R");
|
||||
addBlendShape("LipsStretch_L");
|
||||
addBlendShape("LipsStretch_R");
|
||||
addBlendShape("LipsUpperClose");
|
||||
addBlendShape("LipsLowerClose");
|
||||
addBlendShape("LipsUpperUp");
|
||||
addBlendShape("LipsLowerDown");
|
||||
addBlendShape("LipsUpperOpen");
|
||||
addBlendShape("LipsLowerOpen");
|
||||
addBlendShape("LipsFunnel");
|
||||
addBlendShape("LipsPucker");
|
||||
addBlendShape("ChinLowerRaise");
|
||||
addBlendShape("ChinUpperRaise");
|
||||
addBlendShape("Sneer");
|
||||
addBlendShape("Puff");
|
||||
addBlendShape("CheekSquint_L");
|
||||
addBlendShape("CheekSquint_R");
|
||||
|
||||
for (var i = 0; i < allBlendShapes.length; i++) {
|
||||
targetBlendCoefficient[i] = 0;
|
||||
currentBlendCoefficient[i] = 0;
|
||||
}
|
||||
|
||||
function setRandomExpression() {
|
||||
for (var i = 0; i < allBlendShapes.length; i++) {
|
||||
targetBlendCoefficient[i] = Math.random();
|
||||
}
|
||||
}
|
||||
|
||||
var expressionChangeSpeed = 0.1;
|
||||
|
||||
function updateBlendShapes(deltaTime) {
|
||||
|
||||
for (var i = 0; i < allBlendShapes.length; i++) {
|
||||
currentBlendCoefficient[i] += (targetBlendCoefficient[i] - currentBlendCoefficient[i]) * expressionChangeSpeed;
|
||||
Avatar.setBlendshape(allBlendShapes[i], currentBlendCoefficient[i]);
|
||||
}
|
||||
}
|
||||
|
||||
function update(deltaTime) {
|
||||
timePassed += deltaTime;
|
||||
if (timePassed > updateSpeed) {
|
||||
timePassed = 0;
|
||||
setRandomExpression();
|
||||
}
|
||||
updateBlendShapes(deltaTime);
|
||||
}
|
||||
|
||||
Script.update.connect(update);
|
||||
|
|
@ -51,7 +51,6 @@ var lastVoxelScale = 0;
|
|||
var dragStart = { x: 0, y: 0 };
|
||||
var wheelPixelsMoved = 0;
|
||||
|
||||
|
||||
var mouseX = 0;
|
||||
var mouseY = 0;
|
||||
|
||||
|
|
@ -168,7 +167,16 @@ var voxelPreview = Overlays.addOverlay("cube", {
|
|||
lineWidth: 4
|
||||
});
|
||||
|
||||
var linePreviewTop = Overlays.addOverlay("line3d", {
|
||||
var linePreviewTop = [];
|
||||
var linePreviewBottom = [];
|
||||
var linePreviewLeft = [];
|
||||
var linePreviewRight = [];
|
||||
|
||||
// Currend cursor index
|
||||
var currentCursor = 0;
|
||||
|
||||
function addLineOverlay() {
|
||||
return Overlays.addOverlay("line3d", {
|
||||
position: { x: 0, y: 0, z: 0},
|
||||
end: { x: 0, y: 0, z: 0},
|
||||
color: { red: 255, green: 255, blue: 255},
|
||||
|
|
@ -176,34 +184,24 @@ var linePreviewTop = Overlays.addOverlay("line3d", {
|
|||
visible: false,
|
||||
lineWidth: previewLineWidth
|
||||
});
|
||||
}
|
||||
|
||||
//Cursor line previews for up to three cursors
|
||||
linePreviewTop[0] = addLineOverlay();
|
||||
linePreviewTop[1] = addLineOverlay();
|
||||
linePreviewTop[2] = addLineOverlay();
|
||||
|
||||
var linePreviewBottom = Overlays.addOverlay("line3d", {
|
||||
position: { x: 0, y: 0, z: 0},
|
||||
end: { x: 0, y: 0, z: 0},
|
||||
color: { red: 255, green: 255, blue: 255},
|
||||
alpha: 1,
|
||||
visible: false,
|
||||
lineWidth: previewLineWidth
|
||||
});
|
||||
|
||||
var linePreviewLeft = Overlays.addOverlay("line3d", {
|
||||
position: { x: 0, y: 0, z: 0},
|
||||
end: { x: 0, y: 0, z: 0},
|
||||
color: { red: 255, green: 255, blue: 255},
|
||||
alpha: 1,
|
||||
visible: false,
|
||||
lineWidth: previewLineWidth
|
||||
});
|
||||
|
||||
var linePreviewRight = Overlays.addOverlay("line3d", {
|
||||
position: { x: 0, y: 0, z: 0},
|
||||
end: { x: 0, y: 0, z: 0},
|
||||
color: { red: 255, green: 255, blue: 255},
|
||||
alpha: 1,
|
||||
visible: false,
|
||||
lineWidth: previewLineWidth
|
||||
});
|
||||
|
||||
linePreviewBottom[0] = addLineOverlay();
|
||||
linePreviewBottom[1] = addLineOverlay();
|
||||
linePreviewBottom[2] = addLineOverlay();
|
||||
|
||||
linePreviewLeft[0] = addLineOverlay();
|
||||
linePreviewLeft[1] = addLineOverlay();
|
||||
linePreviewLeft[2] = addLineOverlay();
|
||||
|
||||
linePreviewRight[0] = addLineOverlay();
|
||||
linePreviewRight[1] = addLineOverlay();
|
||||
linePreviewRight[2] = addLineOverlay();
|
||||
|
||||
// these will be used below
|
||||
var scaleSelectorWidth = 144;
|
||||
|
|
@ -809,21 +807,21 @@ function showPreviewLines() {
|
|||
var pasteVoxel = getNewPasteVoxel(pickRay);
|
||||
|
||||
// X axis
|
||||
Overlays.editOverlay(linePreviewBottom, {
|
||||
Overlays.editOverlay(linePreviewBottom[currentCursor], {
|
||||
position: pasteVoxel.origin,
|
||||
end: {x: pasteVoxel.origin.x + pasteVoxel.voxelSize, y: pasteVoxel.origin.y, z: pasteVoxel.origin.z },
|
||||
visible: true
|
||||
});
|
||||
|
||||
// Y axis
|
||||
Overlays.editOverlay(linePreviewRight, {
|
||||
Overlays.editOverlay(linePreviewRight[currentCursor], {
|
||||
position: pasteVoxel.origin,
|
||||
end: {x: pasteVoxel.origin.x, y: pasteVoxel.origin.y + pasteVoxel.voxelSize, z: pasteVoxel.origin.z },
|
||||
visible: true
|
||||
});
|
||||
|
||||
// Z axis
|
||||
Overlays.editOverlay(linePreviewTop, {
|
||||
Overlays.editOverlay(linePreviewTop[currentCursor], {
|
||||
position: pasteVoxel.origin,
|
||||
end: {x: pasteVoxel.origin.x, y: pasteVoxel.origin.y, z: pasteVoxel.origin.z - pasteVoxel.voxelSize },
|
||||
visible: true
|
||||
|
|
@ -837,10 +835,10 @@ function showPreviewLines() {
|
|||
if (intersection.intersects) {
|
||||
resultVoxel = calculateVoxelFromIntersection(intersection,"");
|
||||
Overlays.editOverlay(voxelPreview, { visible: false });
|
||||
Overlays.editOverlay(linePreviewTop, { position: resultVoxel.topLeft, end: resultVoxel.topRight, visible: true });
|
||||
Overlays.editOverlay(linePreviewBottom, { position: resultVoxel.bottomLeft, end: resultVoxel.bottomRight, visible: true });
|
||||
Overlays.editOverlay(linePreviewLeft, { position: resultVoxel.topLeft, end: resultVoxel.bottomLeft, visible: true });
|
||||
Overlays.editOverlay(linePreviewRight, { position: resultVoxel.topRight, end: resultVoxel.bottomRight, visible: true });
|
||||
Overlays.editOverlay(linePreviewTop[currentCursor], { position: resultVoxel.topLeft, end: resultVoxel.topRight, visible: true });
|
||||
Overlays.editOverlay(linePreviewBottom[currentCursor], { position: resultVoxel.bottomLeft, end: resultVoxel.bottomRight, visible: true });
|
||||
Overlays.editOverlay(linePreviewLeft[currentCursor], { position: resultVoxel.topLeft, end: resultVoxel.bottomLeft, visible: true });
|
||||
Overlays.editOverlay(linePreviewRight[currentCursor], { position: resultVoxel.topRight, end: resultVoxel.bottomRight, visible: true });
|
||||
colors[0] = {red: intersection.voxel.red, green: intersection.voxel.green , blue: intersection.voxel.blue };
|
||||
|
||||
if (copyScale) {
|
||||
|
|
@ -849,10 +847,10 @@ function showPreviewLines() {
|
|||
moveTools();
|
||||
} else {
|
||||
Overlays.editOverlay(voxelPreview, { visible: false });
|
||||
Overlays.editOverlay(linePreviewTop, { visible: false });
|
||||
Overlays.editOverlay(linePreviewBottom, { visible: false });
|
||||
Overlays.editOverlay(linePreviewLeft, { visible: false });
|
||||
Overlays.editOverlay(linePreviewRight, { visible: false });
|
||||
Overlays.editOverlay(linePreviewTop[currentCursor], { visible: false });
|
||||
Overlays.editOverlay(linePreviewBottom[currentCursor], { visible: false });
|
||||
Overlays.editOverlay(linePreviewLeft[currentCursor], { visible: false });
|
||||
Overlays.editOverlay(linePreviewRight[currentCursor], { visible: false });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -862,20 +860,20 @@ function showPreviewGuides() {
|
|||
showPreviewVoxel();
|
||||
|
||||
// make sure alternative is hidden
|
||||
Overlays.editOverlay(linePreviewTop, { visible: false });
|
||||
Overlays.editOverlay(linePreviewBottom, { visible: false });
|
||||
Overlays.editOverlay(linePreviewLeft, { visible: false });
|
||||
Overlays.editOverlay(linePreviewRight, { visible: false });
|
||||
Overlays.editOverlay(linePreviewTop[currentCursor], { visible: false });
|
||||
Overlays.editOverlay(linePreviewBottom[currentCursor], { visible: false });
|
||||
Overlays.editOverlay(linePreviewLeft[currentCursor], { visible: false });
|
||||
Overlays.editOverlay(linePreviewRight[currentCursor], { visible: false });
|
||||
} else {
|
||||
showPreviewLines();
|
||||
}
|
||||
} else {
|
||||
// make sure all previews are off
|
||||
Overlays.editOverlay(voxelPreview, { visible: false });
|
||||
Overlays.editOverlay(linePreviewTop, { visible: false });
|
||||
Overlays.editOverlay(linePreviewBottom, { visible: false });
|
||||
Overlays.editOverlay(linePreviewLeft, { visible: false });
|
||||
Overlays.editOverlay(linePreviewRight, { visible: false });
|
||||
Overlays.editOverlay(linePreviewTop[currentCursor], { visible: false });
|
||||
Overlays.editOverlay(linePreviewBottom[currentCursor], { visible: false });
|
||||
Overlays.editOverlay(linePreviewLeft[currentCursor], { visible: false });
|
||||
Overlays.editOverlay(linePreviewRight[currentCursor], { visible: false });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -968,6 +966,14 @@ function mousePressEvent(event) {
|
|||
return;
|
||||
}
|
||||
|
||||
if (event.deviceID == 1500) { // Left Hydra Controller
|
||||
currentCursor = 0;
|
||||
} else if (event.deviceID == 1501) { // Right Hydra Controller
|
||||
currentCursor = 1;
|
||||
} else {
|
||||
currentCursor = 2;
|
||||
}
|
||||
|
||||
var clickedOnSomething = false;
|
||||
var clickedOverlay = Overlays.getOverlayAtPoint({x: event.x, y: event.y});
|
||||
|
||||
|
|
@ -1220,6 +1226,7 @@ function menuItemEvent(menuItem) {
|
|||
}
|
||||
|
||||
function mouseMoveEvent(event) {
|
||||
|
||||
if (!editToolsOn) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -1227,6 +1234,14 @@ function mouseMoveEvent(event) {
|
|||
return;
|
||||
}
|
||||
|
||||
if (event.deviceID == 1500) { // Left Hydra Controller
|
||||
currentCursor = 0;
|
||||
} else if (event.deviceID == 1501) { // Right Hydra Controller
|
||||
currentCursor = 1;
|
||||
} else {
|
||||
currentCursor = 2;
|
||||
}
|
||||
|
||||
// Move Import Preview
|
||||
if (isImporting) {
|
||||
var pickRay = Camera.computePickRay(event.x, event.y);
|
||||
|
|
@ -1475,10 +1490,12 @@ Controller.captureKeyEvents({ text: "-" });
|
|||
|
||||
function scriptEnding() {
|
||||
Overlays.deleteOverlay(voxelPreview);
|
||||
Overlays.deleteOverlay(linePreviewTop);
|
||||
Overlays.deleteOverlay(linePreviewBottom);
|
||||
Overlays.deleteOverlay(linePreviewLeft);
|
||||
Overlays.deleteOverlay(linePreviewRight);
|
||||
for (var i = 0; i < linePreviewTop.length; i++) {
|
||||
Overlays.deleteOverlay(linePreviewTop[i]);
|
||||
Overlays.deleteOverlay(linePreviewBottom[i]);
|
||||
Overlays.deleteOverlay(linePreviewLeft[i]);
|
||||
Overlays.deleteOverlay(linePreviewRight[i]);
|
||||
}
|
||||
for (s = 0; s < numColors; s++) {
|
||||
Overlays.deleteOverlay(swatches[s]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ AvatarData::AvatarData() :
|
|||
_handState(0),
|
||||
_keyState(NO_KEY_DOWN),
|
||||
_isChatCirclingEnabled(false),
|
||||
_forceFaceshiftConnected(false),
|
||||
_hasNewJointRotations(true),
|
||||
_headData(NULL),
|
||||
_handData(NULL),
|
||||
|
|
@ -80,6 +81,9 @@ QByteArray AvatarData::toByteArray() {
|
|||
// lazily allocate memory for HeadData in case we're not an Avatar instance
|
||||
if (!_headData) {
|
||||
_headData = new HeadData(this);
|
||||
if (_forceFaceshiftConnected) {
|
||||
_headData->_isFaceshiftConnected = true;
|
||||
}
|
||||
}
|
||||
|
||||
QByteArray avatarDataByteArray;
|
||||
|
|
|
|||
|
|
@ -206,6 +206,10 @@ public:
|
|||
|
||||
Q_INVOKABLE virtual QStringList getJointNames() const { return _jointNames; }
|
||||
|
||||
Q_INVOKABLE void setBlendshape(QString name, float val) { _headData->setBlendshape(name, val); }
|
||||
|
||||
void setForceFaceshiftConnected(bool connected) { _forceFaceshiftConnected = connected; }
|
||||
|
||||
// key state
|
||||
void setKeyState(KeyState s) { _keyState = s; }
|
||||
KeyState keyState() const { return _keyState; }
|
||||
|
|
@ -300,7 +304,7 @@ protected:
|
|||
std::string _chatMessage;
|
||||
|
||||
bool _isChatCirclingEnabled;
|
||||
|
||||
bool _forceFaceshiftConnected;
|
||||
bool _hasNewJointRotations; // set in AvatarData, cleared in Avatar
|
||||
|
||||
HeadData* _headData;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@
|
|||
#include "AvatarData.h"
|
||||
#include "HeadData.h"
|
||||
|
||||
#include "../fbx/src/FBXReader.h"
|
||||
|
||||
HeadData::HeadData(AvatarData* owningAvatar) :
|
||||
_baseYaw(0.0f),
|
||||
_basePitch(0.0f),
|
||||
|
|
@ -52,6 +54,26 @@ void HeadData::setOrientation(const glm::quat& orientation) {
|
|||
_baseRoll = eulers.z;
|
||||
}
|
||||
|
||||
void HeadData::setBlendshape(QString name, float val) {
|
||||
static bool hasInitializedLookupMap = false;
|
||||
static QMap<QString, int> blendshapeLookupMap;
|
||||
//Lazily construct a lookup map from the blendshapes
|
||||
if (!hasInitializedLookupMap) {
|
||||
for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) {
|
||||
blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i;
|
||||
}
|
||||
}
|
||||
|
||||
//Check to see if the named blendshape exists, and then set its value if it does
|
||||
QMap<QString, int>::iterator it = blendshapeLookupMap.find(name);
|
||||
if (it != blendshapeLookupMap.end()) {
|
||||
if (_blendshapeCoefficients.size() <= it.value()) {
|
||||
_blendshapeCoefficients.resize(it.value() + 1);
|
||||
}
|
||||
_blendshapeCoefficients[it.value()] = val;
|
||||
}
|
||||
}
|
||||
|
||||
void HeadData::addYaw(float yaw) {
|
||||
setBaseYaw(_baseYaw + yaw);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ public:
|
|||
float getAudioAverageLoudness() const { return _audioAverageLoudness; }
|
||||
void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; }
|
||||
|
||||
void setBlendshape(QString name, float val);
|
||||
const QVector<float>& getBlendshapeCoefficients() const { return _blendshapeCoefficients; }
|
||||
|
||||
float getPupilDilation() const { return _pupilDilation; }
|
||||
|
|
|
|||
|
|
@ -577,6 +577,8 @@ const char* FACESHIFT_BLENDSHAPES[] = {
|
|||
""
|
||||
};
|
||||
|
||||
const int NUM_FACESHIFT_BLENDSHAPES = sizeof(FACESHIFT_BLENDSHAPES) / sizeof(char*);
|
||||
|
||||
const char* HUMANIK_JOINTS[] = {
|
||||
"RightHand",
|
||||
"RightForeArm",
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ typedef QList<FBXNode> FBXNodeList;
|
|||
|
||||
/// The names of the blendshapes expected by Faceshift, terminated with an empty string.
|
||||
extern const char* FACESHIFT_BLENDSHAPES[];
|
||||
/// The size of FACESHIFT_BLENDSHAPES
|
||||
extern const int NUM_FACESHIFT_BLENDSHAPES;
|
||||
|
||||
/// The names of the joints in the Maya HumanIK rig, terminated with an empty string.
|
||||
extern const char* HUMANIK_JOINTS[];
|
||||
|
|
|
|||
Loading…
Reference in a new issue