Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-08-07 19:41:20 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #4836 from ctrlaltdavid/20535

Browse source 
CR for #20535 - Improve DDE eyelid control
This commit is contained in:
Philip Rosedale 2015-05-12 17:00:02 -07:00
commit 65277e0573
7 changed files with 164 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
interface/src/Application.cpp
View file

@ -1949,6 +1949,7 @@ void Application::setActiveFaceTracker() {
#endif #endif
#ifdef HAVE_DDE #ifdef HAVE_DDE
bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::UseCamera); bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::UseCamera);
Menu::getInstance()->getActionForOption(MenuOption::BinaryEyelidControl)->setVisible(isUsingDDE);
Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE);
Menu::getInstance()->getActionForOption(MenuOption::VelocityFilter)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::VelocityFilter)->setVisible(isUsingDDE);
Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setVisible(isUsingDDE);

2
interface/src/Menu.cpp
View file

@ -388,6 +388,8 @@ Menu::Menu() {
} }
#ifdef HAVE_DDE #ifdef HAVE_DDE
faceTrackingMenu->addSeparator(); faceTrackingMenu->addSeparator();
QAction* binaryEyelidControl = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::BinaryEyelidControl, 0, true);
binaryEyelidControl->setVisible(false);
QAction* useAudioForMouth = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseAudioForMouth, 0, true); QAction* useAudioForMouth = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseAudioForMouth, 0, true);
useAudioForMouth->setVisible(false); useAudioForMouth->setVisible(false);
QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::VelocityFilter, 0, true); QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::VelocityFilter, 0, true);

1
interface/src/Menu.h
View file

@ -149,6 +149,7 @@ namespace MenuOption {
const QString AudioStatsShowInjectedStreams = "Audio Stats Show Injected Streams"; const QString AudioStatsShowInjectedStreams = "Audio Stats Show Injected Streams";
const QString AvatarReceiveStats = "Show Receive Stats"; const QString AvatarReceiveStats = "Show Receive Stats";
const QString BandwidthDetails = "Bandwidth Details"; const QString BandwidthDetails = "Bandwidth Details";
const QString BinaryEyelidControl = "Binary Eyelid Control";
const QString BlueSpeechSphere = "Blue Sphere While Speaking"; const QString BlueSpeechSphere = "Blue Sphere While Speaking";
const QString BookmarkLocation = "Bookmark Location"; const QString BookmarkLocation = "Bookmark Location";
const QString Bookmarks = "Bookmarks"; const QString Bookmarks = "Bookmarks";

114
interface/src/devices/DdeFaceTracker.cpp
View file

@ -137,7 +137,7 @@ struct Packet {
}; };
static const float STARTING_DDE_MESSAGE_TIME = 0.033f; static const float STARTING_DDE_MESSAGE_TIME = 0.033f;
static const float DEFAULT_DDE_EYE_CLOSING_THRESHOLD = 0.8f;
static const int CALIBRATION_SAMPLES = 150; static const int CALIBRATION_SAMPLES = 150;
#ifdef WIN32 #ifdef WIN32
@ -182,6 +182,7 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
_lastEyeBlinks(), _lastEyeBlinks(),
_filteredEyeBlinks(), _filteredEyeBlinks(),
_lastEyeCoefficients(), _lastEyeCoefficients(),
_eyeClosingThreshold("ddeEyeClosingThreshold", DEFAULT_DDE_EYE_CLOSING_THRESHOLD),
_isCalibrating(false), _isCalibrating(false),
_calibrationCount(0), _calibrationCount(0),
_calibrationValues(), _calibrationValues(),
@ -194,8 +195,8 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
_coefficientAverages.resize(NUM_FACESHIFT_BLENDSHAPES); _coefficientAverages.resize(NUM_FACESHIFT_BLENDSHAPES);
_calibrationValues.resize(NUM_FACESHIFT_BLENDSHAPES); _calibrationValues.resize(NUM_FACESHIFT_BLENDSHAPES);
_eyeStates[0] = EYE_OPEN; _eyeStates[0] = EYE_UNCONTROLLED;
_eyeStates[1] = EYE_OPEN; _eyeStates[1] = EYE_UNCONTROLLED;
connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams()));
connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError))); connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError)));
@ -450,57 +451,72 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
// Finesse EyeBlink values // Finesse EyeBlink values
float eyeCoefficients[2]; float eyeCoefficients[2];
for (int i = 0; i < 2; i++) { if (Menu::getInstance()->isOptionChecked(MenuOption::BinaryEyelidControl)) {
// Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen if (_eyeStates[0] == EYE_UNCONTROLLED) {
// -ve values control EyeOpen; +ve values control EyeBlink _eyeStates[0] = EYE_OPEN;
static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value _eyeStates[1] = EYE_OPEN;
eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD);
// Change to closing or opening states
const float EYE_CONTROL_HYSTERISIS = 0.25f;
const float EYE_CLOSING_THRESHOLD = 0.8f;
const float EYE_OPENING_THRESHOLD = EYE_CONTROL_THRESHOLD - EYE_CONTROL_HYSTERISIS;
if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > EYE_CLOSING_THRESHOLD) {
_eyeStates[i] = EYE_CLOSING;
} else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING)
&& eyeCoefficients[i] < EYE_OPENING_THRESHOLD) {
_eyeStates[i] = EYE_OPENING;
} }
const float EYELID_MOVEMENT_RATE = 10.0f; // units/second for (int i = 0; i < 2; i++) {
const float EYE_OPEN_SCALE = 0.2f; // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen
if (_eyeStates[i] == EYE_CLOSING) { // -ve values control EyeOpen; +ve values control EyeBlink
// Close eyelid until it's fully closed static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value
float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime; eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD);
if (closingValue >= 1.0) {
_eyeStates[i] = EYE_CLOSED; // Change to closing or opening states
eyeCoefficients[i] = 1.0; const float EYE_CONTROL_HYSTERISIS = 0.25f;
} else { float eyeClosingThreshold = getEyeClosingThreshold();
eyeCoefficients[i] = closingValue; float eyeOpeningThreshold = eyeClosingThreshold - EYE_CONTROL_HYSTERISIS;
if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > eyeClosingThreshold) {
_eyeStates[i] = EYE_CLOSING;
} else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING)
&& eyeCoefficients[i] < eyeOpeningThreshold) {
_eyeStates[i] = EYE_OPENING;
} }
} else if (_eyeStates[i] == EYE_OPENING) {
// Open eyelid until it meets the current adjusted value const float EYELID_MOVEMENT_RATE = 10.0f; // units/second
float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime; const float EYE_OPEN_SCALE = 0.2f;
if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) { if (_eyeStates[i] == EYE_CLOSING) {
_eyeStates[i] = EYE_OPEN; // Close eyelid until it's fully closed
float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime;
if (closingValue >= 1.0) {
_eyeStates[i] = EYE_CLOSED;
eyeCoefficients[i] = 1.0;
} else {
eyeCoefficients[i] = closingValue;
}
} else if (_eyeStates[i] == EYE_OPENING) {
// Open eyelid until it meets the current adjusted value
float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime;
if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) {
_eyeStates[i] = EYE_OPEN;
eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE;
} else {
eyeCoefficients[i] = openingValue;
}
} else if (_eyeStates[i] == EYE_OPEN) {
// Reduce eyelid movement
eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE;
} else { } else if (_eyeStates[i] == EYE_CLOSED) {
eyeCoefficients[i] = openingValue; // Keep eyelid fully closed
eyeCoefficients[i] = 1.0;
} }
} else if (_eyeStates[i] == EYE_OPEN) {
// Reduce eyelid movement
eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE;
} else if (_eyeStates[i] == EYE_CLOSED) {
// Keep eyelid fully closed
eyeCoefficients[i] = 1.0;
} }
if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) {
// Couple eyelids
eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f;
}
_lastEyeCoefficients[0] = eyeCoefficients[0];
_lastEyeCoefficients[1] = eyeCoefficients[1];
} else {
_eyeStates[0] = EYE_UNCONTROLLED;
_eyeStates[1] = EYE_UNCONTROLLED;
eyeCoefficients[0] = _filteredEyeBlinks[0];
eyeCoefficients[1] = _filteredEyeBlinks[1];
} }
if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) {
// Couple eyelids
eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f;
}
_lastEyeCoefficients[0] = eyeCoefficients[0];
_lastEyeCoefficients[1] = eyeCoefficients[1];
// Use EyeBlink values to control both EyeBlink and EyeOpen // Use EyeBlink values to control both EyeBlink and EyeOpen
if (eyeCoefficients[0] > 0) { if (eyeCoefficients[0] > 0) {
@ -544,6 +560,10 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
} }
} }
void DdeFaceTracker::setEyeClosingThreshold(float eyeClosingThreshold) {
_eyeClosingThreshold.set(eyeClosingThreshold);
}
static const int CALIBRATION_BILLBOARD_WIDTH = 240; static const int CALIBRATION_BILLBOARD_WIDTH = 240;
static const int CALIBRATION_BILLBOARD_HEIGHT = 180; static const int CALIBRATION_BILLBOARD_HEIGHT = 180;
static const int CALIBRATION_BILLBOARD_TOP_MARGIN = 60; static const int CALIBRATION_BILLBOARD_TOP_MARGIN = 60;

9
interface/src/devices/DdeFaceTracker.h
View file

@ -50,6 +50,9 @@ public:
float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); }
float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); }
float getEyeClosingThreshold() { return _eyeClosingThreshold.get(); }
void setEyeClosingThreshold(float eyeClosingThreshold);
public slots: public slots:
void setEnabled(bool enabled); void setEnabled(bool enabled);
void calibrate(); void calibrate();
@ -89,8 +92,7 @@ private:
int _rightBlinkIndex; int _rightBlinkIndex;
int _leftEyeOpenIndex; int _leftEyeOpenIndex;
int _rightEyeOpenIndex; int _rightEyeOpenIndex;
// Brows
int _browDownLeftIndex; int _browDownLeftIndex;
int _browDownRightIndex; int _browDownRightIndex;
int _browUpCenterIndex; int _browUpCenterIndex;
@ -114,6 +116,7 @@ private:
float _filteredBrowUp; float _filteredBrowUp;
enum EyeState { enum EyeState {
EYE_UNCONTROLLED,
EYE_OPEN, EYE_OPEN,
EYE_CLOSING, EYE_CLOSING,
EYE_CLOSED, EYE_CLOSED,
@ -123,6 +126,8 @@ private:
float _lastEyeBlinks[2]; float _lastEyeBlinks[2];
float _filteredEyeBlinks[2]; float _filteredEyeBlinks[2];
float _lastEyeCoefficients[2]; float _lastEyeCoefficients[2];
Setting::Handle<float> _eyeClosingThreshold;
QVector<float> _coefficientAverages; QVector<float> _coefficientAverages;
bool _isCalibrating; bool _isCalibrating;

9
interface/src/ui/PreferencesDialog.cpp
View file

@ -14,6 +14,7 @@
#include <AudioClient.h> #include <AudioClient.h>
#include <avatar/AvatarManager.h> #include <avatar/AvatarManager.h>
#include <devices/DdeFaceTracker.h>
#include <devices/Faceshift.h> #include <devices/Faceshift.h>
#include <devices/SixenseManager.h> #include <devices/SixenseManager.h>
#include <NetworkingConstants.h> #include <NetworkingConstants.h>
@ -135,6 +136,10 @@ void PreferencesDialog::loadPreferences() {
ui.pupilDilationSlider->setValue(myAvatar->getHead()->getPupilDilation() * ui.pupilDilationSlider->setValue(myAvatar->getHead()->getPupilDilation() *
ui.pupilDilationSlider->maximum()); ui.pupilDilationSlider->maximum());
auto dde = DependencyManager::get<DdeFaceTracker>();
ui.ddeEyeClosingThresholdSlider->setValue(dde->getEyeClosingThreshold() *
ui.ddeEyeClosingThresholdSlider->maximum());
auto faceshift = DependencyManager::get<Faceshift>(); auto faceshift = DependencyManager::get<Faceshift>();
ui.faceshiftEyeDeflectionSider->setValue(faceshift->getEyeDeflection() * ui.faceshiftEyeDeflectionSider->setValue(faceshift->getEyeDeflection() *
ui.faceshiftEyeDeflectionSider->maximum()); ui.faceshiftEyeDeflectionSider->maximum());
@ -222,6 +227,10 @@ void PreferencesDialog::savePreferences() {
qApp->setFieldOfView(ui.fieldOfViewSpin->value()); qApp->setFieldOfView(ui.fieldOfViewSpin->value());
auto dde = DependencyManager::get<DdeFaceTracker>();
dde->setEyeClosingThreshold(ui.ddeEyeClosingThresholdSlider->value() /
(float)ui.ddeEyeClosingThresholdSlider->maximum());
auto faceshift = DependencyManager::get<Faceshift>(); auto faceshift = DependencyManager::get<Faceshift>();
faceshift->setEyeDeflection(ui.faceshiftEyeDeflectionSider->value() / faceshift->setEyeDeflection(ui.faceshiftEyeDeflectionSider->value() /
(float)ui.faceshiftEyeDeflectionSider->maximum()); (float)ui.faceshiftEyeDeflectionSider->maximum());

78
interface/ui/preferencesDialog.ui
View file

@ -1256,7 +1256,7 @@
</font> </font>
</property> </property>
<property name="text"> <property name="text">
<string>Pupil dillation</string> <string>Pupil dilation</string>
</property> </property>
<property name="indent"> <property name="indent">
<number>0</number> <number>0</number>
@ -1310,6 +1310,82 @@
</item> </item>
</layout> </layout>
</item> </item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_28">
<property name="spacing">
<number>0</number>
</property>
<property name="topMargin">
<number>7</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<number>7</number>
</property>
<item alignment="Qt::AlignLeft">
<widget class="QLabel" name="label_7">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="text">
<string>Camera binary eyelid threshold</string>
</property>
<property name="indent">
<number>0</number>
</property>
<property name="buddy">
<cstring>ddeEyeClosingThresholdSlider</cstring>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_8">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QSlider" name="ddeEyeClosingThresholdSlider">
<property name="sizePolicy">
<sizepolicy hsizetype="Fixed" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="minimumSize">
<size>
<width>130</width>
<height>0</height>
</size>
</property>
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
</widget>
</item>
</layout>
</item>
<item> <item>
<layout class="QHBoxLayout" name="horizontalLayout_6"> <layout class="QHBoxLayout" name="horizontalLayout_6">
<property name="spacing"> <property name="spacing">