Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-20 12:04:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #5348 from ctrlaltdavid/20602

Browse source 
DO NOT MERGE - QA for #20602 - Fix and improve eye saccades
This commit is contained in:
Philip Rosedale 2015-07-23 17:42:24 -07:00
commit a1544956e3
11 changed files with 103 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
interface/src/Application.cpp
View file

@ -2318,23 +2318,21 @@ void Application::updateMyAvatarLookAtPosition() {
lookAtSpot = _myAvatar->getHead()->getEyePosition() +
(_myAvatar->getHead()->getFinalOrientationInWorldFrame() * glm::vec3(0.0f, 0.0f, -TREE_SCALE));
}
}
// Deflect the eyes a bit to match the detected gaze from Faceshift if active.
// DDE doesn't track eyes.
if (tracker && typeid(*tracker) == typeid(Faceshift) && !tracker->isMuted()) {
float eyePitch = tracker->getEstimatedEyePitch();
float eyeYaw = tracker->getEstimatedEyeYaw();
const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f;
glm::vec3 origin = _myAvatar->getHead()->getEyePosition();
float pitchSign = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? -1.0f : 1.0f;
float deflection = DependencyManager::get<Faceshift>()->getEyeDeflection();
if (isLookingAtSomeone) {
deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT;
}
lookAtSpot = origin + _myCamera.getRotation() * glm::quat(glm::radians(glm::vec3(
eyePitch * pitchSign * deflection, eyeYaw * deflection, 0.0f))) *
// Deflect the eyes a bit to match the detected gaze from the face tracker if active.
if (tracker && !tracker->isMuted()) {
float eyePitch = tracker->getEstimatedEyePitch();
float eyeYaw = tracker->getEstimatedEyeYaw();
const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f;
glm::vec3 origin = _myAvatar->getHead()->getEyePosition();
float deflection = tracker->getEyeDeflection();
if (isLookingAtSomeone) {
deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT;
}
lookAtSpot = origin + _myCamera.getRotation() * glm::quat(glm::radians(glm::vec3(
eyePitch * deflection, eyeYaw * deflection, 0.0f))) *
glm::inverse(_myCamera.getRotation()) * (lookAtSpot - origin);
}
}
_myAvatar->getHead()->setLookAtPosition(lookAtSpot);

4
interface/src/avatar/FaceModel.cpp
View file

@ -72,8 +72,8 @@ void FaceModel::maybeUpdateEyeRotation(Model* model, const JointState& parentSta
glm::translate(state.getDefaultTranslationInConstrainedFrame()) *
joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation));
glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getFinalOrientationInWorldFrame() * IDENTITY_FRONT, 0.0f));
glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getCorrectedLookAtPosition() +
_owningHead->getSaccade() - model->getTranslation(), 1.0f));
glm::vec3 lookAtDelta = _owningHead->getCorrectedLookAtPosition() - model->getTranslation();
glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(lookAtDelta + glm::length(lookAtDelta) * _owningHead->getSaccade(), 1.0f));
glm::quat between = rotationBetween(front, lookAt);
const float MAX_ANGLE = 30.0f * RADIANS_PER_DEGREE;
state.setRotationInConstrainedFrame(glm::angleAxis(glm::clamp(glm::angle(between), -MAX_ANGLE, MAX_ANGLE), glm::axis(between)) *

6
interface/src/avatar/Head.cpp
View file

@ -129,13 +129,14 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) {
const float AVERAGE_SACCADE_INTERVAL = 6.0f;
const float MICROSACCADE_MAGNITUDE = 0.002f;
const float SACCADE_MAGNITUDE = 0.04f;
const float NOMINAL_FRAME_RATE = 60.0f;
if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) {
_saccadeTarget = MICROSACCADE_MAGNITUDE * randVector();
} else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) {
_saccadeTarget = SACCADE_MAGNITUDE * randVector();
}
_saccade += (_saccadeTarget - _saccade) * 0.5f;
_saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime);
// Detect transition from talking to not; force blink after that and a delay
bool forceBlink = false;
@ -342,7 +343,8 @@ glm::quat Head::getCameraOrientation() const {
glm::quat Head::getEyeRotation(const glm::vec3& eyePosition) const {
glm::quat orientation = getOrientation();
return rotationBetween(orientation * IDENTITY_FRONT, _lookAtPosition + _saccade - eyePosition) * orientation;
glm::vec3 lookAtDelta = _lookAtPosition - eyePosition;
return rotationBetween(orientation * IDENTITY_FRONT, lookAtDelta + glm::length(lookAtDelta) * _saccade) * orientation;
}
glm::vec3 Head::getScalePivot() const {

45
interface/src/devices/DdeFaceTracker.cpp
View file

@ -157,6 +157,10 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
_reset(false),
_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
_rightBlinkIndex(1),
_leftEyeDownIndex(4),
_rightEyeDownIndex(5),
_leftEyeInIndex(6),
_rightEyeInIndex(7),
_leftEyeOpenIndex(8),
_rightEyeOpenIndex(9),
_browDownLeftIndex(14),
@ -173,6 +177,14 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
_filteredHeadTranslation(glm::vec3(0.0f)),
_lastBrowUp(0.0f),
_filteredBrowUp(0.0f),
_eyePitch(0.0f),
_eyeYaw(0.0f),
_lastEyePitch(0.0f),
_lastEyeYaw(0.0f),
_filteredEyePitch(0.0f),
_filteredEyeYaw(0.0f),
_longTermAverageEyePitch(0.0f),
_longTermAverageEyeYaw(0.0f),
_lastEyeBlinks(),
_filteredEyeBlinks(),
_lastEyeCoefficients(),
@ -282,6 +294,17 @@ void DdeFaceTracker::reset() {
}
}
void DdeFaceTracker::update(float deltaTime) {
if (!isActive()) {
return;
}
FaceTracker::update(deltaTime);
glm::vec3 headEulers = glm::degrees(glm::eulerAngles(_headRotation));
_estimatedEyePitch = _eyePitch - headEulers.x;
_estimatedEyeYaw = _eyeYaw - headEulers.y;
}
bool DdeFaceTracker::isActive() const {
return (_ddeProcess != NULL);
}
@ -436,6 +459,28 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
_coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD;
_coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD;
// Eye pitch and yaw
// EyeDown coefficients work better over both +ve and -ve values than EyeUp values.
// EyeIn coefficients work better over both +ve and -ve values than EyeOut values.
// Pitch and yaw values are relative to the screen.
const float EYE_PITCH_SCALE = -1500.0f; // Sign, scale, and average to be similar to Faceshift values.
_eyePitch = EYE_PITCH_SCALE * (_coefficients[_leftEyeDownIndex] + _coefficients[_rightEyeDownIndex]);
const float EYE_YAW_SCALE = 2000.0f; // Scale and average to be similar to Faceshift values.
_eyeYaw = EYE_YAW_SCALE * (_coefficients[_leftEyeInIndex] + _coefficients[_rightEyeInIndex]);
if (isFiltering) {
const float EYE_VELOCITY_FILTER_STRENGTH = 0.005f;
float pitchVelocity = fabsf(_eyePitch - _lastEyePitch) / _averageMessageTime;
float pitchVelocityFilter = glm::clamp(pitchVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);
_filteredEyePitch = pitchVelocityFilter * _eyePitch + (1.0f - pitchVelocityFilter) * _filteredEyePitch;
_lastEyePitch = _eyePitch;
_eyePitch = _filteredEyePitch;
float yawVelocity = fabsf(_eyeYaw - _lastEyeYaw) / _averageMessageTime;
float yawVelocityFilter = glm::clamp(yawVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);
_filteredEyeYaw = yawVelocityFilter * _eyeYaw + (1.0f - yawVelocityFilter) * _filteredEyeYaw;
_lastEyeYaw = _eyeYaw;
_eyeYaw = _filteredEyeYaw;
}
// Velocity filter EyeBlink values
const float DDE_EYEBLINK_SCALE = 3.0f;
float eyeBlinks[] = { DDE_EYEBLINK_SCALE * _coefficients[_leftBlinkIndex],

16
interface/src/devices/DdeFaceTracker.h
View file

@ -31,6 +31,7 @@ class DdeFaceTracker : public FaceTracker, public Dependency {
public:
virtual void init();
virtual void reset();
virtual void update(float deltaTime);
virtual bool isActive() const;
virtual bool isTracking() const;
@ -93,6 +94,11 @@ private:
int _leftEyeOpenIndex;
int _rightEyeOpenIndex;
int _leftEyeDownIndex;
int _rightEyeDownIndex;
int _leftEyeInIndex;
int _rightEyeInIndex;
int _browDownLeftIndex;
int _browDownRightIndex;
int _browUpCenterIndex;
@ -115,6 +121,16 @@ private:
float _lastBrowUp;
float _filteredBrowUp;
float _eyePitch; // Degrees, relative to screen
float _eyeYaw;
float _lastEyePitch;
float _lastEyeYaw;
float _filteredEyePitch;
float _filteredEyeYaw;
float _longTermAverageEyePitch = 0.0f;
float _longTermAverageEyeYaw = 0.0f;
bool _longTermAverageInitialized = false;
enum EyeState {
EYE_UNCONTROLLED,
EYE_OPEN,

7
interface/src/devices/FaceTracker.cpp
View file

@ -20,6 +20,9 @@
const int FPS_TIMER_DELAY = 2000; // ms
const int FPS_TIMER_DURATION = 2000; // ms
const float DEFAULT_EYE_DEFLECTION = 0.25f;
Setting::Handle<float> FaceTracker::_eyeDeflection("faceshiftEyeDeflection", DEFAULT_EYE_DEFLECTION);
void FaceTracker::init() {
_isMuted = Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking);
_isInitialized = true; // FaceTracker can be used now
@ -106,3 +109,7 @@ void FaceTracker::toggleMute() {
_isMuted = !_isMuted;
emit muteToggled();
}
void FaceTracker::setEyeDeflection(float eyeDeflection) {
_eyeDeflection.set(eyeDeflection);
}

7
interface/src/devices/FaceTracker.h
View file

@ -18,6 +18,8 @@
#include <glm/glm.hpp>
#include <glm/gtc/quaternion.hpp>
#include <SettingHandle.h>
/// Base class for face trackers (Faceshift, DDE).
class FaceTracker : public QObject {
Q_OBJECT
@ -47,6 +49,9 @@ public:
void setIsMuted(bool isMuted) { _isMuted = isMuted; }
void toggleMute();
static float getEyeDeflection() { return _eyeDeflection.get(); }
static void setEyeDeflection(float eyeDeflection);
signals:
void muteToggled();
@ -77,6 +82,8 @@ private slots:
private:
bool _isCalculatingFPS = false;
int _frameCount = 0;
static Setting::Handle<float> _eyeDeflection;
};
#endif // hifi_FaceTracker_h

6
interface/src/devices/Faceshift.cpp
View file

@ -28,10 +28,8 @@ using namespace std;
const QString DEFAULT_FACESHIFT_HOSTNAME = "localhost";
const quint16 FACESHIFT_PORT = 33433;
const float DEFAULT_FACESHIFT_EYE_DEFLECTION = 0.25f;
Faceshift::Faceshift() :
_eyeDeflection("faceshiftEyeDeflection", DEFAULT_FACESHIFT_EYE_DEFLECTION),
_hostname("faceshiftHostname", DEFAULT_FACESHIFT_HOSTNAME)
{
#ifdef HAVE_FACESHIFT
@ -306,10 +304,6 @@ void Faceshift::receive(const QByteArray& buffer) {
FaceTracker::countFrame();
}
void Faceshift::setEyeDeflection(float faceshiftEyeDeflection) {
_eyeDeflection.set(faceshiftEyeDeflection);
}
void Faceshift::setHostname(const QString& hostname) {
_hostname.set(hostname);
}

4
interface/src/devices/Faceshift.h
View file

@ -68,9 +68,6 @@ public:
float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); }
float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); }
float getEyeDeflection() { return _eyeDeflection.get(); }
void setEyeDeflection(float faceshiftEyeDeflection);
QString getHostname() { return _hostname.get(); }
void setHostname(const QString& hostname);
@ -134,7 +131,6 @@ private:
float _longTermAverageEyeYaw = 0.0f;
bool _longTermAverageInitialized = false;
Setting::Handle<float> _eyeDeflection;
Setting::Handle<QString> _hostname;
// see http://support.faceshift.com/support/articles/35129-export-of-blendshapes

12
interface/src/ui/PreferencesDialog.cpp
View file

@ -142,10 +142,10 @@ void PreferencesDialog::loadPreferences() {
ui.ddeEyeClosingThresholdSlider->setValue(dde->getEyeClosingThreshold() *
ui.ddeEyeClosingThresholdSlider->maximum());
auto faceshift = DependencyManager::get<Faceshift>();
ui.faceshiftEyeDeflectionSider->setValue(faceshift->getEyeDeflection() *
ui.faceshiftEyeDeflectionSider->maximum());
ui.faceTrackerEyeDeflectionSider->setValue(FaceTracker::getEyeDeflection() *
ui.faceTrackerEyeDeflectionSider->maximum());
auto faceshift = DependencyManager::get<Faceshift>();
ui.faceshiftHostnameEdit->setText(faceshift->getHostname());
auto audio = DependencyManager::get<AudioClient>();
@ -233,10 +233,10 @@ void PreferencesDialog::savePreferences() {
dde->setEyeClosingThreshold(ui.ddeEyeClosingThresholdSlider->value() /
(float)ui.ddeEyeClosingThresholdSlider->maximum());
auto faceshift = DependencyManager::get<Faceshift>();
faceshift->setEyeDeflection(ui.faceshiftEyeDeflectionSider->value() /
(float)ui.faceshiftEyeDeflectionSider->maximum());
FaceTracker::setEyeDeflection(ui.faceTrackerEyeDeflectionSider->value() /
(float)ui.faceTrackerEyeDeflectionSider->maximum());
auto faceshift = DependencyManager::get<Faceshift>();
faceshift->setHostname(ui.faceshiftHostnameEdit->text());
qApp->setMaxOctreePacketsPerSecond(ui.maxOctreePPSSpin->value());

6
interface/ui/preferencesDialog.ui
View file

@ -1468,13 +1468,13 @@
</font>
</property>
<property name="text">
<string>Faceshift eye deflection</string>
<string>Face tracker eye deflection</string>
</property>
<property name="indent">
<number>0</number>
</property>
<property name="buddy">
<cstring>faceshiftEyeDeflectionSider</cstring>
<cstring>faceTrackerEyeDeflectionSider</cstring>
</property>
</widget>
</item>
@ -1497,7 +1497,7 @@
</spacer>
</item>
<item>
<widget class="QSlider" name="faceshiftEyeDeflectionSider">
<widget class="QSlider" name="faceTrackerEyeDeflectionSider">
<property name="sizePolicy">
<sizepolicy hsizetype="Fixed" vsizetype="Fixed">
<horstretch>0</horstretch>