From 8eb4abc49cb8c23c344be69a2e50e813b6b32bf2 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 26 Feb 2015 19:14:09 +0100 Subject: [PATCH 1/9] Rework FaceTracker --- interface/src/devices/FaceTracker.cpp | 5 ++--- interface/src/devices/FaceTracker.h | 24 +++++++++++++++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/interface/src/devices/FaceTracker.cpp b/interface/src/devices/FaceTracker.cpp index 52fe04de77..8d61789715 100644 --- a/interface/src/devices/FaceTracker.cpp +++ b/interface/src/devices/FaceTracker.cpp @@ -11,7 +11,6 @@ #include "FaceTracker.h" -FaceTracker::FaceTracker() : - _estimatedEyePitch(0.0f), - _estimatedEyeYaw(0.0f) { +inline float FaceTracker::getBlendshapeCoefficient(int index) const { + return isValidBlendshapeIndex(index) ? _blendshapeCoefficients[index] : 0.0f; } diff --git a/interface/src/devices/FaceTracker.h b/interface/src/devices/FaceTracker.h index 7c367b7899..07c479e3e3 100644 --- a/interface/src/devices/FaceTracker.h +++ b/interface/src/devices/FaceTracker.h @@ -18,13 +18,17 @@ #include #include -/// Base class for face trackers (Faceshift, Visage). +/// Base class for face trackers (Faceshift, Visage, DDE). class FaceTracker : public QObject { Q_OBJECT public: - FaceTracker(); - virtual ~FaceTracker() {} + virtual bool isActive() const { return false; } + virtual bool isTracking() const { return false; } + + virtual void init() {} + virtual void update(float deltaTime) {} + virtual void reset() {} const glm::vec3& getHeadTranslation() const { return _headTranslation; } const glm::quat& getHeadRotation() const { return _headRotation; } @@ -32,15 +36,21 @@ public: float getEstimatedEyePitch() const { return _estimatedEyePitch; } float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } + int getNumBlendshapes() const { return _blendshapeCoefficients.size(); } + bool isValidBlendshapeIndex(int index) const { return index >= 0 && index < getNumBlendshapes(); } const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + float getBlendshapeCoefficient(int index) const; protected: - glm::vec3 _headTranslation; - glm::quat _headRotation; - float _estimatedEyePitch; - float _estimatedEyeYaw; + + glm::vec3 _headTranslation = glm::vec3(0.0f); + glm::quat _headRotation = glm::quat(); + float _estimatedEyePitch = 0.0f; + float _estimatedEyeYaw = 0.0f; QVector _blendshapeCoefficients; + + float _fadeCoefficient = 0.0f; }; #endif // hifi_FaceTracker_h From b6968a6b1538875504b4f90b6d71038b13b75850 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 26 Feb 2015 19:15:49 +0100 Subject: [PATCH 2/9] Update Facetrackers --- interface/src/devices/DdeFaceTracker.cpp | 18 +---- interface/src/devices/DdeFaceTracker.h | 10 ++- interface/src/devices/Faceshift.cpp | 73 ++++++------------- interface/src/devices/Faceshift.h | 89 ++++++++++++------------ interface/src/devices/Visage.cpp | 3 +- interface/src/devices/Visage.h | 10 +-- 6 files changed, 77 insertions(+), 126 deletions(-) diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index 87a180bd1e..ed0e4057ef 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -50,6 +50,8 @@ DdeFaceTracker::DdeFaceTracker() : } DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 port) : + _host(host), + _port(port), _lastReceiveTimestamp(0), _reset(false), _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes @@ -63,9 +65,7 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 port) : _browUpRightIndex(18), _mouthSmileLeftIndex(28), _mouthSmileRightIndex(29), - _jawOpenIndex(21), - _host(host), - _port(port) + _jawOpenIndex(21) { _blendshapeCoefficients.resize(NUM_EXPRESSION); @@ -80,18 +80,6 @@ DdeFaceTracker::~DdeFaceTracker() { } } -void DdeFaceTracker::init() { - -} - -void DdeFaceTracker::reset() { - _reset = true; -} - -void DdeFaceTracker::update() { - -} - void DdeFaceTracker::setEnabled(bool enabled) { // isOpen() does not work as one might expect on QUdpSocket; don't test isOpen() before closing socket. _udpSocket.close(); diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index bd5d066732..88528370a2 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -23,12 +23,10 @@ class DdeFaceTracker : public FaceTracker, public Dependency { SINGLETON_DEPENDENCY public: - //initialization - void init(); - void reset(); - void update(); - - bool isActive() const; + virtual void reset() { _reset = true; } + + virtual bool isActive() const; + virtual bool isTracking() const { return isActive(); } float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 2f541c1c4e..87ee0ef31c 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -25,37 +25,11 @@ using namespace fs; using namespace std; +const QString DEFAULT_FACESHIFT_HOSTNAME = "localhost"; const quint16 FACESHIFT_PORT = 33433; -float STARTING_FACESHIFT_FRAME_TIME = 0.033f; +const float DEFAULT_FACESHIFT_EYE_DEFLECTION = 0.25f; Faceshift::Faceshift() : - _tcpEnabled(true), - _tcpRetryCount(0), - _lastTrackingStateReceived(0), - _averageFrameTime(STARTING_FACESHIFT_FRAME_TIME), - _headAngularVelocity(0), - _headLinearVelocity(0), - _lastHeadTranslation(0), - _filteredHeadTranslation(0), - _eyeGazeLeftPitch(0.0f), - _eyeGazeLeftYaw(0.0f), - _eyeGazeRightPitch(0.0f), - _eyeGazeRightYaw(0.0f), - _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes - _rightBlinkIndex(1), - _leftEyeOpenIndex(8), - _rightEyeOpenIndex(9), - _browDownLeftIndex(14), - _browDownRightIndex(15), - _browUpCenterIndex(16), - _browUpLeftIndex(17), - _browUpRightIndex(18), - _mouthSmileLeftIndex(28), - _mouthSmileRightIndex(29), - _jawOpenIndex(21), - _longTermAverageEyePitch(0.0f), - _longTermAverageEyeYaw(0.0f), - _longTermAverageInitialized(false), _eyeDeflection("faceshiftEyeDeflection", DEFAULT_FACESHIFT_EYE_DEFLECTION), _hostname("faceshiftHostname", DEFAULT_FACESHIFT_HOSTNAME) { @@ -71,31 +45,15 @@ Faceshift::Faceshift() : #endif } +#ifdef HAVE_FACESHIFT void Faceshift::init() { -#ifdef HAVE_FACESHIFT setTCPEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift)); -#endif } -bool Faceshift::isConnectedOrConnecting() const { - return _tcpSocket.state() == QAbstractSocket::ConnectedState || - (_tcpRetryCount == 0 && _tcpSocket.state() != QAbstractSocket::UnconnectedState); -} - -bool Faceshift::isActive() const { -#ifdef HAVE_FACESHIFT - const quint64 ACTIVE_TIMEOUT_USECS = 1000000; - return (usecTimestampNow() - _lastTrackingStateReceived) < ACTIVE_TIMEOUT_USECS; -#else - return false; -#endif -} - -void Faceshift::update() { +void Faceshift::update(float deltaTime) { if (!isActive()) { return; } - PerformanceTimer perfTimer("faceshift"); // get the euler angles relative to the window glm::vec3 eulers = glm::degrees(safeEulerAngles(_headRotation * glm::quat(glm::radians(glm::vec3( (_eyeGazeLeftPitch + _eyeGazeRightPitch) / 2.0f, (_eyeGazeLeftYaw + _eyeGazeRightYaw) / 2.0f, 0.0f))))); @@ -116,14 +74,28 @@ void Faceshift::update() { } void Faceshift::reset() { -#ifdef HAVE_FACESHIFT if (_tcpSocket.state() == QAbstractSocket::ConnectedState) { string message; fsBinaryStream::encode_message(message, fsMsgCalibrateNeutral()); send(message); } _longTermAverageInitialized = false; +} + +bool Faceshift::isActive() const { + const quint64 ACTIVE_TIMEOUT_USECS = 1000000; + return (usecTimestampNow() - _lastTrackingStateReceived) < ACTIVE_TIMEOUT_USECS; +} + +bool Faceshift::isTracking() const { + return isActive() && _tracking; +} #endif + + +bool Faceshift::isConnectedOrConnecting() const { + return _tcpSocket.state() == QAbstractSocket::ConnectedState || + (_tcpRetryCount == 0 && _tcpSocket.state() != QAbstractSocket::UnconnectedState); } void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, @@ -148,12 +120,13 @@ void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float } void Faceshift::setTCPEnabled(bool enabled) { +#ifdef HAVE_FACESHIFT if ((_tcpEnabled = enabled)) { connectSocket(); - } else { _tcpSocket.disconnectFromHost(); } +#endif } void Faceshift::connectSocket() { @@ -202,10 +175,6 @@ void Faceshift::readFromSocket() { receive(_tcpSocket.readAll()); } -float Faceshift::getBlendshapeCoefficient(int index) const { - return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; -} - void Faceshift::send(const std::string& message) { _tcpSocket.write(message.data(), message.size()); } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 0cef3d3e13..f224448b8e 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -24,8 +24,7 @@ #include "FaceTracker.h" -const float DEFAULT_FACESHIFT_EYE_DEFLECTION = 0.25f; -const QString DEFAULT_FACESHIFT_HOSTNAME = "localhost"; +const float STARTING_FACESHIFT_FRAME_TIME = 0.033f; /// Handles interaction with the Faceshift software, which provides head position/orientation and facial features. class Faceshift : public FaceTracker, public Dependency { @@ -33,11 +32,17 @@ class Faceshift : public FaceTracker, public Dependency { SINGLETON_DEPENDENCY public: - void init(); +#ifdef HAVE_FACESHIFT + // If we don't have faceshift, use the base class' methods + virtual void init(); + virtual void update(float deltaTime); + virtual void reset(); - bool isConnectedOrConnecting() const; + virtual bool isActive() const; + virtual bool isTracking() const; +#endif - bool isActive() const; + bool isConnectedOrConnecting() const; const glm::vec3& getHeadAngularVelocity() const { return _headAngularVelocity; } @@ -68,9 +73,6 @@ public: QString getHostname() { return _hostname.get(); } void setHostname(const QString& hostname); - - void update(); - void reset(); void updateFakeCoefficients(float leftBlink, float rightBlink, @@ -82,15 +84,12 @@ public: QVector& coefficients) const; signals: - void connectionStateChanged(); public slots: - void setTCPEnabled(bool enabled); private slots: - void connectSocket(); void noteConnected(); void noteError(QAbstractSocket::SocketError error); @@ -101,8 +100,6 @@ private: Faceshift(); virtual ~Faceshift() {} - float getBlendshapeCoefficient(int index) const; - void send(const std::string& message); void receive(const QByteArray& buffer); @@ -113,48 +110,48 @@ private: fs::fsBinaryStream _stream; #endif - bool _tcpEnabled; - int _tcpRetryCount; - bool _tracking; - quint64 _lastTrackingStateReceived; - float _averageFrameTime; + bool _tcpEnabled = true; + int _tcpRetryCount = 0; + bool _tracking = false; + quint64 _lastTrackingStateReceived = 0; + float _averageFrameTime = STARTING_FACESHIFT_FRAME_TIME; - glm::vec3 _headAngularVelocity; - glm::vec3 _headLinearVelocity; - glm::vec3 _lastHeadTranslation; - glm::vec3 _filteredHeadTranslation; + glm::vec3 _headAngularVelocity = glm::vec3(0.0f); + glm::vec3 _headLinearVelocity = glm::vec3(0.0f); + glm::vec3 _lastHeadTranslation = glm::vec3(0.0f); + glm::vec3 _filteredHeadTranslation = glm::vec3(0.0f); // degrees - float _eyeGazeLeftPitch; - float _eyeGazeLeftYaw; - float _eyeGazeRightPitch; - float _eyeGazeRightYaw; - - int _leftBlinkIndex; - int _rightBlinkIndex; - int _leftEyeOpenIndex; - int _rightEyeOpenIndex; - - // Brows - int _browDownLeftIndex; - int _browDownRightIndex; - int _browUpCenterIndex; - int _browUpLeftIndex; - int _browUpRightIndex; - - int _mouthSmileLeftIndex; - int _mouthSmileRightIndex; - - int _jawOpenIndex; + float _eyeGazeLeftPitch = 0.0f; + float _eyeGazeLeftYaw = 0.0f; + float _eyeGazeRightPitch = 0.0f; + float _eyeGazeRightYaw = 0.0f; // degrees - float _longTermAverageEyePitch; - float _longTermAverageEyeYaw; - bool _longTermAverageInitialized; + float _longTermAverageEyePitch = 0.0f; + float _longTermAverageEyeYaw = 0.0f; + bool _longTermAverageInitialized = false; Setting::Handle _eyeDeflection; Setting::Handle _hostname; + // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes + int _leftBlinkIndex = 0; + int _rightBlinkIndex = 1; + int _leftEyeOpenIndex = 8; + int _rightEyeOpenIndex = 9; + + // Brows + int _browDownLeftIndex = 14; + int _browDownRightIndex = 15; + int _browUpCenterIndex = 16; + int _browUpLeftIndex = 17; + int _browUpRightIndex = 18; + + int _mouthSmileLeftIndex = 28; + int _mouthSmileRightIndex = 29; + + int _jawOpenIndex = 21; }; #endif // hifi_Faceshift_h diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index 39bda83e61..45c19a6ab3 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -126,13 +126,12 @@ void Visage::init() { updateEnabled(); } -void Visage::update() { +void Visage::update(float deltaTime) { #ifdef HAVE_VISAGE _active = (_tracker->getTrackingData(_data) == TRACK_STAT_OK); if (!_active) { return; } - PerformanceTimer perfTimer("visage"); _headRotation = glm::quat(glm::vec3(-_data->faceRotation[0], -_data->faceRotation[1], _data->faceRotation[2])); _headTranslation = (glm::vec3(_data->faceTranslation[0], _data->faceTranslation[1], _data->faceTranslation[2]) - _headOrigin) * TRANSLATION_SCALE; diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h index 690a9f3066..cd2ec3ecad 100644 --- a/interface/src/devices/Visage.h +++ b/interface/src/devices/Visage.h @@ -31,12 +31,12 @@ class Visage : public FaceTracker, public Dependency { SINGLETON_DEPENDENCY public: - void init(); + virtual void init(); + virtual void update(float deltaTime); + virtual void reset(); - bool isActive() const { return _active; } - - void update(); - void reset(); + virtual bool isActive() const { return _active; } + virtual bool isTracking() const { return isActive(); } public slots: From f560f97c6b5c1ed0b3381a8be5439843caaeec51 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 26 Feb 2015 19:18:05 +0100 Subject: [PATCH 3/9] Only update active FaceTracker --- interface/src/Application.cpp | 36 ++++------------------------------- interface/src/Application.h | 3 --- 2 files changed, 4 insertions(+), 35 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 0bdd8f8f53..e1b4aa40ea 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1859,35 +1859,6 @@ void Application::updateMouseRay() { } } -void Application::updateFaceshift() { - bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); - PerformanceWarning warn(showWarnings, "Application::updateFaceshift()"); - auto faceshift = DependencyManager::get(); - // Update faceshift - faceshift->update(); - - // Copy angular velocity if measured by faceshift, to the head - if (faceshift->isActive()) { - _myAvatar->getHead()->setAngularVelocity(faceshift->getHeadAngularVelocity()); - } -} - -void Application::updateVisage() { - bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); - PerformanceWarning warn(showWarnings, "Application::updateVisage()"); - - // Update Visage - DependencyManager::get()->update(); -} - -void Application::updateDDE() { - bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); - PerformanceWarning warn(showWarnings, "Application::updateDDE()"); - - // Update Cara - DependencyManager::get()->update(); -} - void Application::updateMyAvatarLookAtPosition() { PerformanceTimer perfTimer("lookAt"); bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); @@ -2067,12 +2038,13 @@ void Application::update(float deltaTime) { { PerformanceTimer perfTimer("devices"); DeviceTracker::updateAll(); - updateFaceshift(); - updateVisage(); + FaceTracker* tracker = getActiveFaceTracker(); + if (tracker) { + tracker->update(deltaTime); + } SixenseManager::getInstance().update(deltaTime); JoystickScriptingInterface::getInstance().update(); _prioVR.update(deltaTime); - } // Dispatch input events diff --git a/interface/src/Application.h b/interface/src/Application.h index 9cd19c8259..4a8e712d30 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -421,9 +421,6 @@ private: // Various helper functions called during update() void updateLOD(); void updateMouseRay(); - void updateFaceshift(); - void updateVisage(); - void updateDDE(); void updateMyAvatarLookAtPosition(); void updateThreads(float deltaTime); void updateMetavoxels(float deltaTime); From 682b3f5e61c109a659e8f2ea11a2926b92dc6049 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 26 Feb 2015 19:18:52 +0100 Subject: [PATCH 4/9] Remove unused angular velocity --- interface/src/avatar/Head.cpp | 1 - interface/src/avatar/Head.h | 4 ---- 2 files changed, 5 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 52f1b3ee86..a6458c7a22 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -42,7 +42,6 @@ Head::Head(Avatar* owningAvatar) : _mouth2(0.0f), _mouth3(0.0f), _mouth4(0.0f), - _angularVelocity(0,0,0), _renderLookatVectors(false), _saccade(0.0f, 0.0f, 0.0f), _saccadeTarget(0.0f, 0.0f, 0.0f), diff --git a/interface/src/avatar/Head.h b/interface/src/avatar/Head.h index 89eea35903..6c53c2d7aa 100644 --- a/interface/src/avatar/Head.h +++ b/interface/src/avatar/Head.h @@ -55,9 +55,6 @@ public: /// \return orientationBody * orientationBasePitch glm::quat getCameraOrientation () const; - - const glm::vec3& getAngularVelocity() const { return _angularVelocity; } - void setAngularVelocity(glm::vec3 angularVelocity) { _angularVelocity = angularVelocity; } void setCorrectedLookAtPosition(glm::vec3 correctedLookAtPosition); glm::vec3 getCorrectedLookAtPosition(); @@ -130,7 +127,6 @@ private: float _mouth2; float _mouth3; float _mouth4; - glm::vec3 _angularVelocity; bool _renderLookatVectors; glm::vec3 _saccade; glm::vec3 _saccadeTarget; From e37ef226fc6b04378d2d8ce48960dc49d5cd4312 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 26 Feb 2015 19:21:14 +0100 Subject: [PATCH 5/9] Simplify lookForward compute --- interface/src/avatar/MyAvatar.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index ba628ea0d4..9cdd38dbc8 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -892,13 +892,7 @@ void MyAvatar::updateLookAtTargetAvatar() { _lookAtTargetAvatar.clear(); _targetAvatarPosition = glm::vec3(0.0f); - glm::quat faceRotation = Application::getInstance()->getViewFrustum()->getOrientation(); - FaceTracker* tracker = Application::getInstance()->getActiveFaceTracker(); - if (tracker) { - // If faceshift or other face tracker in use, add on the actual angle of the head - faceRotation *= tracker->getHeadRotation(); - } - glm::vec3 lookForward = faceRotation * IDENTITY_FRONT; + glm::vec3 lookForward = getHead()->getFinalOrientationInWorldFrame() * IDENTITY_FRONT; glm::vec3 cameraPosition = Application::getInstance()->getCamera()->getPosition(); float smallestAngleTo = glm::radians(Application::getInstance()->getCamera()->getFieldOfView()) / 2.0f; From fdf9fdd8778d38263c4c3c17d1228ddeb9e32e9c Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 26 Feb 2015 20:05:47 +0100 Subject: [PATCH 6/9] Relax head/lean when not tracking --- interface/src/avatar/MyAvatar.cpp | 9 +------- interface/src/devices/FaceTracker.cpp | 30 +++++++++++++++++++++++++++ interface/src/devices/FaceTracker.h | 12 +++++------ interface/src/devices/Faceshift.cpp | 2 ++ interface/src/devices/Visage.cpp | 12 +++++------ interface/src/devices/Visage.h | 6 +++--- 6 files changed, 48 insertions(+), 23 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 9cdd38dbc8..8c4ea73775 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -1158,15 +1158,8 @@ void MyAvatar::updateOrientation(float deltaTime) { pitch *= DEGREES_PER_RADIAN; roll *= DEGREES_PER_RADIAN; - // Record the angular velocity - Head* head = getHead(); - if (deltaTime > 0.0f) { - glm::vec3 angularVelocity(pitch - head->getBasePitch(), yaw - head->getBaseYaw(), roll - head->getBaseRoll()); - angularVelocity *= 1.0f / deltaTime; - head->setAngularVelocity(angularVelocity); - } - //Invert yaw and roll when in mirror mode + Head* head = getHead(); if (Application::getInstance()->getCamera()->getMode() == CAMERA_MODE_MIRROR) { head->setBaseYaw(-yaw); head->setBasePitch(pitch); diff --git a/interface/src/devices/FaceTracker.cpp b/interface/src/devices/FaceTracker.cpp index 8d61789715..e570e39345 100644 --- a/interface/src/devices/FaceTracker.cpp +++ b/interface/src/devices/FaceTracker.cpp @@ -14,3 +14,33 @@ inline float FaceTracker::getBlendshapeCoefficient(int index) const { return isValidBlendshapeIndex(index) ? _blendshapeCoefficients[index] : 0.0f; } + +float FaceTracker::getFadeCoefficient() const { + // TODO: apply exponential relaxation + return _relaxationStatus; +} + +const glm::vec3 FaceTracker::getHeadTranslation() const { + return glm::mix(glm::vec3(0.0f), _headTranslation, getFadeCoefficient()); +} + +const glm::quat FaceTracker::getHeadRotation() const { + return glm::mix(glm::quat(), _headRotation, getFadeCoefficient()); +} + +void FaceTracker::update(float deltaTime) { + static const float RELAXATION_TIME = 0.4; // sec + + if (isTracking()) { + if (_relaxationStatus == 1.0f) { + return; + } + _relaxationStatus += deltaTime / RELAXATION_TIME; + } else { + if (_relaxationStatus == 0.0f) { + return; + } + _relaxationStatus -= deltaTime / RELAXATION_TIME; + } + _relaxationStatus = glm::clamp(_relaxationStatus, 0.0f, 1.0f); +} \ No newline at end of file diff --git a/interface/src/devices/FaceTracker.h b/interface/src/devices/FaceTracker.h index 07c479e3e3..89ba1f22f6 100644 --- a/interface/src/devices/FaceTracker.h +++ b/interface/src/devices/FaceTracker.h @@ -27,11 +27,13 @@ public: virtual bool isTracking() const { return false; } virtual void init() {} - virtual void update(float deltaTime) {} + virtual void update(float deltaTime); virtual void reset() {} - const glm::vec3& getHeadTranslation() const { return _headTranslation; } - const glm::quat& getHeadRotation() const { return _headRotation; } + float getFadeCoefficient() const; + + const glm::vec3 getHeadTranslation() const; + const glm::quat getHeadRotation() const; float getEstimatedEyePitch() const { return _estimatedEyePitch; } float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } @@ -42,15 +44,13 @@ public: float getBlendshapeCoefficient(int index) const; protected: - - glm::vec3 _headTranslation = glm::vec3(0.0f); glm::quat _headRotation = glm::quat(); float _estimatedEyePitch = 0.0f; float _estimatedEyeYaw = 0.0f; QVector _blendshapeCoefficients; - float _fadeCoefficient = 0.0f; + float _relaxationStatus = 0.0f; // Between 0.0f and 1.0f }; #endif // hifi_FaceTracker_h diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 87ee0ef31c..a2bb4e74a9 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -54,6 +54,8 @@ void Faceshift::update(float deltaTime) { if (!isActive()) { return; } + FaceTracker::update(deltaTime); + // get the euler angles relative to the window glm::vec3 eulers = glm::degrees(safeEulerAngles(_headRotation * glm::quat(glm::radians(glm::vec3( (_eyeGazeLeftPitch + _eyeGazeRightPitch) / 2.0f, (_eyeGazeLeftYaw + _eyeGazeRightYaw) / 2.0f, 0.0f))))); diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index 45c19a6ab3..010b872bc6 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -41,7 +41,6 @@ const glm::vec3 DEFAULT_HEAD_ORIGIN(0.0f, 0.0f, 0.7f); Visage::Visage() : _enabled(false), - _active(false), _headOrigin(DEFAULT_HEAD_ORIGIN) { #ifdef HAVE_VISAGE @@ -119,19 +118,20 @@ static const QMultiHash >& getActionUnitNameMap() } #endif -const float TRANSLATION_SCALE = 20.0f; +#ifdef HAVE_VISAGE +const float TRANSLATION_SCALE = 20.0f; void Visage::init() { connect(DependencyManager::get().data(), SIGNAL(connectionStateChanged()), SLOT(updateEnabled())); updateEnabled(); } void Visage::update(float deltaTime) { -#ifdef HAVE_VISAGE - _active = (_tracker->getTrackingData(_data) == TRACK_STAT_OK); - if (!_active) { + if (!isActive()) { return; } + FaceTracker::update(deltaTime); + _headRotation = glm::quat(glm::vec3(-_data->faceRotation[0], -_data->faceRotation[1], _data->faceRotation[2])); _headTranslation = (glm::vec3(_data->faceTranslation[0], _data->faceTranslation[1], _data->faceTranslation[2]) - _headOrigin) * TRANSLATION_SCALE; @@ -163,12 +163,12 @@ void Visage::update(float deltaTime) { } _blendshapeCoefficients[leftEyeBlinkIndex] = 1.0f - _data->eyeClosure[1]; _blendshapeCoefficients[rightEyeBlinkIndex] = 1.0f - _data->eyeClosure[0]; -#endif } void Visage::reset() { _headOrigin += _headTranslation / TRANSLATION_SCALE; } +#endif void Visage::updateEnabled() { setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Visage) && diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h index cd2ec3ecad..3ff1ea8c27 100644 --- a/interface/src/devices/Visage.h +++ b/interface/src/devices/Visage.h @@ -31,15 +31,16 @@ class Visage : public FaceTracker, public Dependency { SINGLETON_DEPENDENCY public: +#ifdef HAVE_VISAGE virtual void init(); virtual void update(float deltaTime); virtual void reset(); - virtual bool isActive() const { return _active; } + virtual bool isActive() const { return _tracker->getTrackingData(_data) == TRACK_STAT_OK; } virtual bool isTracking() const { return isActive(); } +#endif public slots: - void updateEnabled(); private: @@ -55,7 +56,6 @@ private: void setEnabled(bool enabled); bool _enabled; - bool _active; glm::vec3 _headOrigin; }; From 9849efe0130082673622349ffc45559e68beafb4 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 27 Feb 2015 15:33:24 +0100 Subject: [PATCH 7/9] Exponential relaxation --- interface/src/devices/FaceTracker.cpp | 13 +++++++------ interface/src/devices/FaceTracker.h | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/interface/src/devices/FaceTracker.cpp b/interface/src/devices/FaceTracker.cpp index e570e39345..f1ab35859b 100644 --- a/interface/src/devices/FaceTracker.cpp +++ b/interface/src/devices/FaceTracker.cpp @@ -16,8 +16,7 @@ inline float FaceTracker::getBlendshapeCoefficient(int index) const { } float FaceTracker::getFadeCoefficient() const { - // TODO: apply exponential relaxation - return _relaxationStatus; + return _fadeCoefficient; } const glm::vec3 FaceTracker::getHeadTranslation() const { @@ -29,18 +28,20 @@ const glm::quat FaceTracker::getHeadRotation() const { } void FaceTracker::update(float deltaTime) { - static const float RELAXATION_TIME = 0.4; // sec + static const float RELAXATION_TIME = 1.0f; // sec + static const float LAMBDA = 1.5; if (isTracking()) { if (_relaxationStatus == 1.0f) { return; } - _relaxationStatus += deltaTime / RELAXATION_TIME; + _relaxationStatus = glm::clamp(_relaxationStatus + deltaTime / RELAXATION_TIME, 0.0f, 1.0f); + _fadeCoefficient = std::exp(-LAMBDA * _relaxationStatus); } else { if (_relaxationStatus == 0.0f) { return; } - _relaxationStatus -= deltaTime / RELAXATION_TIME; + _relaxationStatus = glm::clamp(_relaxationStatus - deltaTime / RELAXATION_TIME, 0.0f, 1.0f); + _fadeCoefficient = 1.0f - std::exp(-LAMBDA * (1.0f - _relaxationStatus)); } - _relaxationStatus = glm::clamp(_relaxationStatus, 0.0f, 1.0f); } \ No newline at end of file diff --git a/interface/src/devices/FaceTracker.h b/interface/src/devices/FaceTracker.h index 89ba1f22f6..1ec0468dec 100644 --- a/interface/src/devices/FaceTracker.h +++ b/interface/src/devices/FaceTracker.h @@ -51,6 +51,7 @@ protected: QVector _blendshapeCoefficients; float _relaxationStatus = 0.0f; // Between 0.0f and 1.0f + float _fadeCoefficient = 0.0f; // Between 0.0f and 1.0f }; #endif // hifi_FaceTracker_h From 8615bb7cff3a39a37cd50301977e35c58b4bd6d0 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 27 Feb 2015 15:58:28 +0100 Subject: [PATCH 8/9] Make relaxation go to EPSILON within TIME --- interface/src/devices/FaceTracker.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/interface/src/devices/FaceTracker.cpp b/interface/src/devices/FaceTracker.cpp index f1ab35859b..d6ccc38a45 100644 --- a/interface/src/devices/FaceTracker.cpp +++ b/interface/src/devices/FaceTracker.cpp @@ -28,20 +28,25 @@ const glm::quat FaceTracker::getHeadRotation() const { } void FaceTracker::update(float deltaTime) { - static const float RELAXATION_TIME = 1.0f; // sec + // Based on exponential distributions: http://en.wikipedia.org/wiki/Exponential_distribution + static const float EPSILON = 0.02f; static const float LAMBDA = 1.5; + static const float INVERSE_AT_EPSILON = -std::log(EPSILON) / LAMBDA; // So that f(1) = EPSILON ~ 0 + static const float RELAXATION_TIME = 0.8f; // sec if (isTracking()) { if (_relaxationStatus == 1.0f) { + _fadeCoefficient = 1.0f; return; } _relaxationStatus = glm::clamp(_relaxationStatus + deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = std::exp(-LAMBDA * _relaxationStatus); + _fadeCoefficient = 1.0f - std::exp(-LAMBDA * _relaxationStatus * INVERSE_AT_EPSILON); } else { if (_relaxationStatus == 0.0f) { + _fadeCoefficient = 0.0f; return; } _relaxationStatus = glm::clamp(_relaxationStatus - deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = 1.0f - std::exp(-LAMBDA * (1.0f - _relaxationStatus)); + _fadeCoefficient = std::exp(-LAMBDA * (1.0f - _relaxationStatus) * INVERSE_AT_EPSILON); } } \ No newline at end of file From db90fcdb6220a7a377cd816d765631230dcbc0a5 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 27 Feb 2015 16:09:15 +0100 Subject: [PATCH 9/9] Simplify expression --- interface/src/devices/FaceTracker.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/interface/src/devices/FaceTracker.cpp b/interface/src/devices/FaceTracker.cpp index d6ccc38a45..d3e0600057 100644 --- a/interface/src/devices/FaceTracker.cpp +++ b/interface/src/devices/FaceTracker.cpp @@ -29,9 +29,8 @@ const glm::quat FaceTracker::getHeadRotation() const { void FaceTracker::update(float deltaTime) { // Based on exponential distributions: http://en.wikipedia.org/wiki/Exponential_distribution - static const float EPSILON = 0.02f; - static const float LAMBDA = 1.5; - static const float INVERSE_AT_EPSILON = -std::log(EPSILON) / LAMBDA; // So that f(1) = EPSILON ~ 0 + static const float EPSILON = 0.02f; // MUST BE < 1.0f + static const float INVERSE_AT_EPSILON = -std::log(EPSILON); // So that f(1.0f) = EPSILON ~ 0.0f static const float RELAXATION_TIME = 0.8f; // sec if (isTracking()) { @@ -40,13 +39,13 @@ void FaceTracker::update(float deltaTime) { return; } _relaxationStatus = glm::clamp(_relaxationStatus + deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = 1.0f - std::exp(-LAMBDA * _relaxationStatus * INVERSE_AT_EPSILON); + _fadeCoefficient = 1.0f - std::exp(-_relaxationStatus * INVERSE_AT_EPSILON); } else { if (_relaxationStatus == 0.0f) { _fadeCoefficient = 0.0f; return; } _relaxationStatus = glm::clamp(_relaxationStatus - deltaTime / RELAXATION_TIME, 0.0f, 1.0f); - _fadeCoefficient = std::exp(-LAMBDA * (1.0f - _relaxationStatus) * INVERSE_AT_EPSILON); + _fadeCoefficient = std::exp(-(1.0f - _relaxationStatus) * INVERSE_AT_EPSILON); } } \ No newline at end of file