Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-20 03:44:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #4348 from Atlante45/relax_face_tracker

Browse source 
Relax face tracker
This commit is contained in:
Philip Rosedale 2015-02-27 08:37:49 -08:00
commit 1663fcb489
13 changed files with 151 additions and 202 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
interface/src/Application.cpp
View file

@ -1859,35 +1859,6 @@ void Application::updateMouseRay() {
}
}
void Application::updateFaceshift() {
bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings);
PerformanceWarning warn(showWarnings, "Application::updateFaceshift()");
auto faceshift = DependencyManager::get<Faceshift>();
// Update faceshift
faceshift->update();
// Copy angular velocity if measured by faceshift, to the head
if (faceshift->isActive()) {
_myAvatar->getHead()->setAngularVelocity(faceshift->getHeadAngularVelocity());
}
}
void Application::updateVisage() {
bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings);
PerformanceWarning warn(showWarnings, "Application::updateVisage()");
// Update Visage
DependencyManager::get<Visage>()->update();
}
void Application::updateDDE() {
bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings);
PerformanceWarning warn(showWarnings, "Application::updateDDE()");
// Update Cara
DependencyManager::get<DdeFaceTracker>()->update();
}
void Application::updateMyAvatarLookAtPosition() {
PerformanceTimer perfTimer("lookAt");
bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings);
@ -2067,12 +2038,13 @@ void Application::update(float deltaTime) {
{
PerformanceTimer perfTimer("devices");
DeviceTracker::updateAll();
updateFaceshift();
updateVisage();
FaceTracker* tracker = getActiveFaceTracker();
if (tracker) {
tracker->update(deltaTime);
}
SixenseManager::getInstance().update(deltaTime);
JoystickScriptingInterface::getInstance().update();
_prioVR.update(deltaTime);
}
// Dispatch input events

3
interface/src/Application.h
View file

@ -421,9 +421,6 @@ private:
// Various helper functions called during update()
void updateLOD();
void updateMouseRay();
void updateFaceshift();
void updateVisage();
void updateDDE();
void updateMyAvatarLookAtPosition();
void updateThreads(float deltaTime);
void updateMetavoxels(float deltaTime);

1
interface/src/avatar/Head.cpp
View file

@ -42,7 +42,6 @@ Head::Head(Avatar* owningAvatar) :
_mouth2(0.0f),
_mouth3(0.0f),
_mouth4(0.0f),
_angularVelocity(0,0,0),
_renderLookatVectors(false),
_saccade(0.0f, 0.0f, 0.0f),
_saccadeTarget(0.0f, 0.0f, 0.0f),

4
interface/src/avatar/Head.h
View file

@ -55,9 +55,6 @@ public:
/// \return orientationBody * orientationBasePitch
glm::quat getCameraOrientation () const;
const glm::vec3& getAngularVelocity() const { return _angularVelocity; }
void setAngularVelocity(glm::vec3 angularVelocity) { _angularVelocity = angularVelocity; }
void setCorrectedLookAtPosition(glm::vec3 correctedLookAtPosition);
glm::vec3 getCorrectedLookAtPosition();
@ -130,7 +127,6 @@ private:
float _mouth2;
float _mouth3;
float _mouth4;
glm::vec3 _angularVelocity;
bool _renderLookatVectors;
glm::vec3 _saccade;
glm::vec3 _saccadeTarget;

17
interface/src/avatar/MyAvatar.cpp
View file

@ -892,13 +892,7 @@ void MyAvatar::updateLookAtTargetAvatar() {
_lookAtTargetAvatar.clear();
_targetAvatarPosition = glm::vec3(0.0f);
glm::quat faceRotation = Application::getInstance()->getViewFrustum()->getOrientation();
FaceTracker* tracker = Application::getInstance()->getActiveFaceTracker();
if (tracker) {
// If faceshift or other face tracker in use, add on the actual angle of the head
faceRotation *= tracker->getHeadRotation();
}
glm::vec3 lookForward = faceRotation * IDENTITY_FRONT;
glm::vec3 lookForward = getHead()->getFinalOrientationInWorldFrame() * IDENTITY_FRONT;
glm::vec3 cameraPosition = Application::getInstance()->getCamera()->getPosition();
float smallestAngleTo = glm::radians(Application::getInstance()->getCamera()->getFieldOfView()) / 2.0f;
@ -1164,15 +1158,8 @@ void MyAvatar::updateOrientation(float deltaTime) {
pitch *= DEGREES_PER_RADIAN;
roll *= DEGREES_PER_RADIAN;
// Record the angular velocity
Head* head = getHead();
if (deltaTime > 0.0f) {
glm::vec3 angularVelocity(pitch - head->getBasePitch(), yaw - head->getBaseYaw(), roll - head->getBaseRoll());
angularVelocity *= 1.0f / deltaTime;
head->setAngularVelocity(angularVelocity);
}
//Invert yaw and roll when in mirror mode
Head* head = getHead();
if (Application::getInstance()->getCamera()->getMode() == CAMERA_MODE_MIRROR) {
head->setBaseYaw(-yaw);
head->setBasePitch(pitch);

18
interface/src/devices/DdeFaceTracker.cpp
View file

@ -50,6 +50,8 @@ DdeFaceTracker::DdeFaceTracker() :
}
DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 port) :
_host(host),
_port(port),
_lastReceiveTimestamp(0),
_reset(false),
_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
@ -63,9 +65,7 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 port) :
_browUpRightIndex(18),
_mouthSmileLeftIndex(28),
_mouthSmileRightIndex(29),
_jawOpenIndex(21),
_host(host),
_port(port)
_jawOpenIndex(21)
{
_blendshapeCoefficients.resize(NUM_EXPRESSION);
@ -80,18 +80,6 @@ DdeFaceTracker::~DdeFaceTracker() {
}
}
void DdeFaceTracker::init() {
}
void DdeFaceTracker::reset() {
_reset = true;
}
void DdeFaceTracker::update() {
}
void DdeFaceTracker::setEnabled(bool enabled) {
// isOpen() does not work as one might expect on QUdpSocket; don't test isOpen() before closing socket.
_udpSocket.close();

10
interface/src/devices/DdeFaceTracker.h
View file

@ -23,12 +23,10 @@ class DdeFaceTracker : public FaceTracker, public Dependency {
SINGLETON_DEPENDENCY
public:
//initialization
void init();
void reset();
void update();
bool isActive() const;
virtual void reset() { _reset = true; }
virtual bool isActive() const;
virtual bool isTracking() const { return isActive(); }
float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); }
float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); }

40
interface/src/devices/FaceTracker.cpp
View file

@ -11,7 +11,41 @@
#include "FaceTracker.h"
FaceTracker::FaceTracker() :
_estimatedEyePitch(0.0f),
_estimatedEyeYaw(0.0f) {
inline float FaceTracker::getBlendshapeCoefficient(int index) const {
return isValidBlendshapeIndex(index) ? _blendshapeCoefficients[index] : 0.0f;
}
float FaceTracker::getFadeCoefficient() const {
return _fadeCoefficient;
}
const glm::vec3 FaceTracker::getHeadTranslation() const {
return glm::mix(glm::vec3(0.0f), _headTranslation, getFadeCoefficient());
}
const glm::quat FaceTracker::getHeadRotation() const {
return glm::mix(glm::quat(), _headRotation, getFadeCoefficient());
}
void FaceTracker::update(float deltaTime) {
// Based on exponential distributions: http://en.wikipedia.org/wiki/Exponential_distribution
static const float EPSILON = 0.02f; // MUST BE < 1.0f
static const float INVERSE_AT_EPSILON = -std::log(EPSILON); // So that f(1.0f) = EPSILON ~ 0.0f
static const float RELAXATION_TIME = 0.8f; // sec
if (isTracking()) {
if (_relaxationStatus == 1.0f) {
_fadeCoefficient = 1.0f;
return;
}
_relaxationStatus = glm::clamp(_relaxationStatus + deltaTime / RELAXATION_TIME, 0.0f, 1.0f);
_fadeCoefficient = 1.0f - std::exp(-_relaxationStatus * INVERSE_AT_EPSILON);
} else {
if (_relaxationStatus == 0.0f) {
_fadeCoefficient = 0.0f;
return;
}
_relaxationStatus = glm::clamp(_relaxationStatus - deltaTime / RELAXATION_TIME, 0.0f, 1.0f);
_fadeCoefficient = std::exp(-(1.0f - _relaxationStatus) * INVERSE_AT_EPSILON);
}
}

31
interface/src/devices/FaceTracker.h
View file

@ -18,29 +18,40 @@
#include <glm/glm.hpp>
#include <glm/gtc/quaternion.hpp>
/// Base class for face trackers (Faceshift, Visage).
/// Base class for face trackers (Faceshift, Visage, DDE).
class FaceTracker : public QObject {
Q_OBJECT
public:
FaceTracker();
virtual ~FaceTracker() {}
virtual bool isActive() const { return false; }
virtual bool isTracking() const { return false; }
const glm::vec3& getHeadTranslation() const { return _headTranslation; }
const glm::quat& getHeadRotation() const { return _headRotation; }
virtual void init() {}
virtual void update(float deltaTime);
virtual void reset() {}
float getFadeCoefficient() const;
const glm::vec3 getHeadTranslation() const;
const glm::quat getHeadRotation() const;
float getEstimatedEyePitch() const { return _estimatedEyePitch; }
float getEstimatedEyeYaw() const { return _estimatedEyeYaw; }
int getNumBlendshapes() const { return _blendshapeCoefficients.size(); }
bool isValidBlendshapeIndex(int index) const { return index >= 0 && index < getNumBlendshapes(); }
const QVector<float>& getBlendshapeCoefficients() const { return _blendshapeCoefficients; }
float getBlendshapeCoefficient(int index) const;
protected:
glm::vec3 _headTranslation;
glm::quat _headRotation;
float _estimatedEyePitch;
float _estimatedEyeYaw;
glm::vec3 _headTranslation = glm::vec3(0.0f);
glm::quat _headRotation = glm::quat();
float _estimatedEyePitch = 0.0f;
float _estimatedEyeYaw = 0.0f;
QVector<float> _blendshapeCoefficients;
float _relaxationStatus = 0.0f; // Between 0.0f and 1.0f
float _fadeCoefficient = 0.0f; // Between 0.0f and 1.0f
};
#endif // hifi_FaceTracker_h

75
interface/src/devices/Faceshift.cpp
View file

@ -25,37 +25,11 @@ using namespace fs;
using namespace std;
const QString DEFAULT_FACESHIFT_HOSTNAME = "localhost";
const quint16 FACESHIFT_PORT = 33433;
float STARTING_FACESHIFT_FRAME_TIME = 0.033f;
const float DEFAULT_FACESHIFT_EYE_DEFLECTION = 0.25f;
Faceshift::Faceshift() :
_tcpEnabled(true),
_tcpRetryCount(0),
_lastTrackingStateReceived(0),
_averageFrameTime(STARTING_FACESHIFT_FRAME_TIME),
_headAngularVelocity(0),
_headLinearVelocity(0),
_lastHeadTranslation(0),
_filteredHeadTranslation(0),
_eyeGazeLeftPitch(0.0f),
_eyeGazeLeftYaw(0.0f),
_eyeGazeRightPitch(0.0f),
_eyeGazeRightYaw(0.0f),
_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
_rightBlinkIndex(1),
_leftEyeOpenIndex(8),
_rightEyeOpenIndex(9),
_browDownLeftIndex(14),
_browDownRightIndex(15),
_browUpCenterIndex(16),
_browUpLeftIndex(17),
_browUpRightIndex(18),
_mouthSmileLeftIndex(28),
_mouthSmileRightIndex(29),
_jawOpenIndex(21),
_longTermAverageEyePitch(0.0f),
_longTermAverageEyeYaw(0.0f),
_longTermAverageInitialized(false),
_eyeDeflection("faceshiftEyeDeflection", DEFAULT_FACESHIFT_EYE_DEFLECTION),
_hostname("faceshiftHostname", DEFAULT_FACESHIFT_HOSTNAME)
{
@ -71,31 +45,17 @@ Faceshift::Faceshift() :
#endif
}
#ifdef HAVE_FACESHIFT
void Faceshift::init() {
#ifdef HAVE_FACESHIFT
setTCPEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift));
#endif
}
bool Faceshift::isConnectedOrConnecting() const {
return _tcpSocket.state() == QAbstractSocket::ConnectedState ||
(_tcpRetryCount == 0 && _tcpSocket.state() != QAbstractSocket::UnconnectedState);
}
bool Faceshift::isActive() const {
#ifdef HAVE_FACESHIFT
const quint64 ACTIVE_TIMEOUT_USECS = 1000000;
return (usecTimestampNow() - _lastTrackingStateReceived) < ACTIVE_TIMEOUT_USECS;
#else
return false;
#endif
}
void Faceshift::update() {
void Faceshift::update(float deltaTime) {
if (!isActive()) {
return;
}
PerformanceTimer perfTimer("faceshift");
FaceTracker::update(deltaTime);
// get the euler angles relative to the window
glm::vec3 eulers = glm::degrees(safeEulerAngles(_headRotation * glm::quat(glm::radians(glm::vec3(
(_eyeGazeLeftPitch + _eyeGazeRightPitch) / 2.0f, (_eyeGazeLeftYaw + _eyeGazeRightYaw) / 2.0f, 0.0f)))));
@ -116,14 +76,28 @@ void Faceshift::update() {
}
void Faceshift::reset() {
#ifdef HAVE_FACESHIFT
if (_tcpSocket.state() == QAbstractSocket::ConnectedState) {
string message;
fsBinaryStream::encode_message(message, fsMsgCalibrateNeutral());
send(message);
}
_longTermAverageInitialized = false;
}
bool Faceshift::isActive() const {
const quint64 ACTIVE_TIMEOUT_USECS = 1000000;
return (usecTimestampNow() - _lastTrackingStateReceived) < ACTIVE_TIMEOUT_USECS;
}
bool Faceshift::isTracking() const {
return isActive() && _tracking;
}
#endif
bool Faceshift::isConnectedOrConnecting() const {
return _tcpSocket.state() == QAbstractSocket::ConnectedState ||
(_tcpRetryCount == 0 && _tcpSocket.state() != QAbstractSocket::UnconnectedState);
}
void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp,
@ -148,12 +122,13 @@ void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float
}
void Faceshift::setTCPEnabled(bool enabled) {
#ifdef HAVE_FACESHIFT
if ((_tcpEnabled = enabled)) {
connectSocket();
} else {
_tcpSocket.disconnectFromHost();
}
#endif
}
void Faceshift::connectSocket() {
@ -202,10 +177,6 @@ void Faceshift::readFromSocket() {
receive(_tcpSocket.readAll());
}
float Faceshift::getBlendshapeCoefficient(int index) const {
return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f;
}
void Faceshift::send(const std::string& message) {
_tcpSocket.write(message.data(), message.size());
}

89
interface/src/devices/Faceshift.h
View file

@ -24,8 +24,7 @@
#include "FaceTracker.h"
const float DEFAULT_FACESHIFT_EYE_DEFLECTION = 0.25f;
const QString DEFAULT_FACESHIFT_HOSTNAME = "localhost";
const float STARTING_FACESHIFT_FRAME_TIME = 0.033f;
/// Handles interaction with the Faceshift software, which provides head position/orientation and facial features.
class Faceshift : public FaceTracker, public Dependency {
@ -33,11 +32,17 @@ class Faceshift : public FaceTracker, public Dependency {
SINGLETON_DEPENDENCY
public:
void init();
#ifdef HAVE_FACESHIFT
// If we don't have faceshift, use the base class' methods
virtual void init();
virtual void update(float deltaTime);
virtual void reset();
bool isConnectedOrConnecting() const;
virtual bool isActive() const;
virtual bool isTracking() const;
#endif
bool isActive() const;
bool isConnectedOrConnecting() const;
const glm::vec3& getHeadAngularVelocity() const { return _headAngularVelocity; }
@ -68,9 +73,6 @@ public:
QString getHostname() { return _hostname.get(); }
void setHostname(const QString& hostname);
void update();
void reset();
void updateFakeCoefficients(float leftBlink,
float rightBlink,
@ -82,15 +84,12 @@ public:
QVector<float>& coefficients) const;
signals:
void connectionStateChanged();
public slots:
void setTCPEnabled(bool enabled);
private slots:
void connectSocket();
void noteConnected();
void noteError(QAbstractSocket::SocketError error);
@ -101,8 +100,6 @@ private:
Faceshift();
virtual ~Faceshift() {}
float getBlendshapeCoefficient(int index) const;
void send(const std::string& message);
void receive(const QByteArray& buffer);
@ -113,48 +110,48 @@ private:
fs::fsBinaryStream _stream;
#endif
bool _tcpEnabled;
int _tcpRetryCount;
bool _tracking;
quint64 _lastTrackingStateReceived;
float _averageFrameTime;
bool _tcpEnabled = true;
int _tcpRetryCount = 0;
bool _tracking = false;
quint64 _lastTrackingStateReceived = 0;
float _averageFrameTime = STARTING_FACESHIFT_FRAME_TIME;
glm::vec3 _headAngularVelocity;
glm::vec3 _headLinearVelocity;
glm::vec3 _lastHeadTranslation;
glm::vec3 _filteredHeadTranslation;
glm::vec3 _headAngularVelocity = glm::vec3(0.0f);
glm::vec3 _headLinearVelocity = glm::vec3(0.0f);
glm::vec3 _lastHeadTranslation = glm::vec3(0.0f);
glm::vec3 _filteredHeadTranslation = glm::vec3(0.0f);
// degrees
float _eyeGazeLeftPitch;
float _eyeGazeLeftYaw;
float _eyeGazeRightPitch;
float _eyeGazeRightYaw;
int _leftBlinkIndex;
int _rightBlinkIndex;
int _leftEyeOpenIndex;
int _rightEyeOpenIndex;
// Brows
int _browDownLeftIndex;
int _browDownRightIndex;
int _browUpCenterIndex;
int _browUpLeftIndex;
int _browUpRightIndex;
int _mouthSmileLeftIndex;
int _mouthSmileRightIndex;
int _jawOpenIndex;
float _eyeGazeLeftPitch = 0.0f;
float _eyeGazeLeftYaw = 0.0f;
float _eyeGazeRightPitch = 0.0f;
float _eyeGazeRightYaw = 0.0f;
// degrees
float _longTermAverageEyePitch;
float _longTermAverageEyeYaw;
bool _longTermAverageInitialized;
float _longTermAverageEyePitch = 0.0f;
float _longTermAverageEyeYaw = 0.0f;
bool _longTermAverageInitialized = false;
Setting::Handle<float> _eyeDeflection;
Setting::Handle<QString> _hostname;
// see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
int _leftBlinkIndex = 0;
int _rightBlinkIndex = 1;
int _leftEyeOpenIndex = 8;
int _rightEyeOpenIndex = 9;
// Brows
int _browDownLeftIndex = 14;
int _browDownRightIndex = 15;
int _browUpCenterIndex = 16;
int _browUpLeftIndex = 17;
int _browUpRightIndex = 18;
int _mouthSmileLeftIndex = 28;
int _mouthSmileRightIndex = 29;
int _jawOpenIndex = 21;
};
#endif // hifi_Faceshift_h

15
interface/src/devices/Visage.cpp
View file

@ -41,7 +41,6 @@ const glm::vec3 DEFAULT_HEAD_ORIGIN(0.0f, 0.0f, 0.7f);
Visage::Visage() :
_enabled(false),
_active(false),
_headOrigin(DEFAULT_HEAD_ORIGIN) {
#ifdef HAVE_VISAGE
@ -119,20 +118,20 @@ static const QMultiHash<QByteArray, QPair<int, float> >& getActionUnitNameMap()
}
#endif
const float TRANSLATION_SCALE = 20.0f;
#ifdef HAVE_VISAGE
const float TRANSLATION_SCALE = 20.0f;
void Visage::init() {
connect(DependencyManager::get<Faceshift>().data(), SIGNAL(connectionStateChanged()), SLOT(updateEnabled()));
updateEnabled();
}
void Visage::update() {
#ifdef HAVE_VISAGE
_active = (_tracker->getTrackingData(_data) == TRACK_STAT_OK);
if (!_active) {
void Visage::update(float deltaTime) {
if (!isActive()) {
return;
}
PerformanceTimer perfTimer("visage");
FaceTracker::update(deltaTime);
_headRotation = glm::quat(glm::vec3(-_data->faceRotation[0], -_data->faceRotation[1], _data->faceRotation[2]));
_headTranslation = (glm::vec3(_data->faceTranslation[0], _data->faceTranslation[1], _data->faceTranslation[2]) -
_headOrigin) * TRANSLATION_SCALE;
@ -164,12 +163,12 @@ void Visage::update() {
}
_blendshapeCoefficients[leftEyeBlinkIndex] = 1.0f - _data->eyeClosure[1];
_blendshapeCoefficients[rightEyeBlinkIndex] = 1.0f - _data->eyeClosure[0];
#endif
}
void Visage::reset() {
_headOrigin += _headTranslation / TRANSLATION_SCALE;
}
#endif
void Visage::updateEnabled() {
setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Visage) &&

14
interface/src/devices/Visage.h
View file

@ -31,15 +31,16 @@ class Visage : public FaceTracker, public Dependency {
SINGLETON_DEPENDENCY
public:
void init();
#ifdef HAVE_VISAGE
virtual void init();
virtual void update(float deltaTime);
virtual void reset();
bool isActive() const { return _active; }
void update();
void reset();
virtual bool isActive() const { return _tracker->getTrackingData(_data) == TRACK_STAT_OK; }
virtual bool isTracking() const { return isActive(); }
#endif
public slots:
void updateEnabled();
private:
@ -55,7 +56,6 @@ private:
void setEnabled(bool enabled);
bool _enabled;
bool _active;
glm::vec3 _headOrigin;
};