diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 01e094e786..89df058e8d 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1949,6 +1949,7 @@ void Application::setActiveFaceTracker() { #endif #ifdef HAVE_DDE bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::UseCamera); + Menu::getInstance()->getActionForOption(MenuOption::BinaryEyelidControl)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::VelocityFilter)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setVisible(isUsingDDE); diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 61213e7334..e1587684cf 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -388,6 +388,8 @@ Menu::Menu() { } #ifdef HAVE_DDE faceTrackingMenu->addSeparator(); + QAction* binaryEyelidControl = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::BinaryEyelidControl, 0, true); + binaryEyelidControl->setVisible(false); QAction* useAudioForMouth = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseAudioForMouth, 0, true); useAudioForMouth->setVisible(false); QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::VelocityFilter, 0, true); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 96c8fedf0d..435df47487 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -149,6 +149,7 @@ namespace MenuOption { const QString AudioStatsShowInjectedStreams = "Audio Stats Show Injected Streams"; const QString AvatarReceiveStats = "Show Receive Stats"; const QString BandwidthDetails = "Bandwidth Details"; + const QString BinaryEyelidControl = "Binary Eyelid Control"; const QString BlueSpeechSphere = "Blue Sphere While Speaking"; const QString BookmarkLocation = "Bookmark Location"; const QString Bookmarks = "Bookmarks"; diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index 14728f021b..e0b888746f 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -137,7 +137,7 @@ struct Packet { }; static const float STARTING_DDE_MESSAGE_TIME = 0.033f; - +static const float DEFAULT_DDE_EYE_CLOSING_THRESHOLD = 0.8f; static const int CALIBRATION_SAMPLES = 150; #ifdef WIN32 @@ -182,6 +182,7 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui _lastEyeBlinks(), _filteredEyeBlinks(), _lastEyeCoefficients(), + _eyeClosingThreshold("ddeEyeClosingThreshold", DEFAULT_DDE_EYE_CLOSING_THRESHOLD), _isCalibrating(false), _calibrationCount(0), _calibrationValues(), @@ -194,8 +195,8 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui _coefficientAverages.resize(NUM_FACESHIFT_BLENDSHAPES); _calibrationValues.resize(NUM_FACESHIFT_BLENDSHAPES); - _eyeStates[0] = EYE_OPEN; - _eyeStates[1] = EYE_OPEN; + _eyeStates[0] = EYE_UNCONTROLLED; + _eyeStates[1] = EYE_UNCONTROLLED; connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError))); @@ -450,57 +451,72 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { // Finesse EyeBlink values float eyeCoefficients[2]; - for (int i = 0; i < 2; i++) { - // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen - // -ve values control EyeOpen; +ve values control EyeBlink - static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value - eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD); - - // Change to closing or opening states - const float EYE_CONTROL_HYSTERISIS = 0.25f; - const float EYE_CLOSING_THRESHOLD = 0.8f; - const float EYE_OPENING_THRESHOLD = EYE_CONTROL_THRESHOLD - EYE_CONTROL_HYSTERISIS; - if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > EYE_CLOSING_THRESHOLD) { - _eyeStates[i] = EYE_CLOSING; - } else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING) - && eyeCoefficients[i] < EYE_OPENING_THRESHOLD) { - _eyeStates[i] = EYE_OPENING; + if (Menu::getInstance()->isOptionChecked(MenuOption::BinaryEyelidControl)) { + if (_eyeStates[0] == EYE_UNCONTROLLED) { + _eyeStates[0] = EYE_OPEN; + _eyeStates[1] = EYE_OPEN; } - const float EYELID_MOVEMENT_RATE = 10.0f; // units/second - const float EYE_OPEN_SCALE = 0.2f; - if (_eyeStates[i] == EYE_CLOSING) { - // Close eyelid until it's fully closed - float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime; - if (closingValue >= 1.0) { - _eyeStates[i] = EYE_CLOSED; - eyeCoefficients[i] = 1.0; - } else { - eyeCoefficients[i] = closingValue; + for (int i = 0; i < 2; i++) { + // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen + // -ve values control EyeOpen; +ve values control EyeBlink + static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value + eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD); + + // Change to closing or opening states + const float EYE_CONTROL_HYSTERISIS = 0.25f; + float eyeClosingThreshold = getEyeClosingThreshold(); + float eyeOpeningThreshold = eyeClosingThreshold - EYE_CONTROL_HYSTERISIS; + if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > eyeClosingThreshold) { + _eyeStates[i] = EYE_CLOSING; + } else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING) + && eyeCoefficients[i] < eyeOpeningThreshold) { + _eyeStates[i] = EYE_OPENING; } - } else if (_eyeStates[i] == EYE_OPENING) { - // Open eyelid until it meets the current adjusted value - float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime; - if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) { - _eyeStates[i] = EYE_OPEN; + + const float EYELID_MOVEMENT_RATE = 10.0f; // units/second + const float EYE_OPEN_SCALE = 0.2f; + if (_eyeStates[i] == EYE_CLOSING) { + // Close eyelid until it's fully closed + float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime; + if (closingValue >= 1.0) { + _eyeStates[i] = EYE_CLOSED; + eyeCoefficients[i] = 1.0; + } else { + eyeCoefficients[i] = closingValue; + } + } else if (_eyeStates[i] == EYE_OPENING) { + // Open eyelid until it meets the current adjusted value + float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime; + if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) { + _eyeStates[i] = EYE_OPEN; + eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; + } else { + eyeCoefficients[i] = openingValue; + } + } else if (_eyeStates[i] == EYE_OPEN) { + // Reduce eyelid movement eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else { - eyeCoefficients[i] = openingValue; + } else if (_eyeStates[i] == EYE_CLOSED) { + // Keep eyelid fully closed + eyeCoefficients[i] = 1.0; } - } else if (_eyeStates[i] == EYE_OPEN) { - // Reduce eyelid movement - eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else if (_eyeStates[i] == EYE_CLOSED) { - // Keep eyelid fully closed - eyeCoefficients[i] = 1.0; } + + if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) { + // Couple eyelids + eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f; + } + + _lastEyeCoefficients[0] = eyeCoefficients[0]; + _lastEyeCoefficients[1] = eyeCoefficients[1]; + } else { + _eyeStates[0] = EYE_UNCONTROLLED; + _eyeStates[1] = EYE_UNCONTROLLED; + + eyeCoefficients[0] = _filteredEyeBlinks[0]; + eyeCoefficients[1] = _filteredEyeBlinks[1]; } - if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) { - // Couple eyelids - eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f; - } - _lastEyeCoefficients[0] = eyeCoefficients[0]; - _lastEyeCoefficients[1] = eyeCoefficients[1]; // Use EyeBlink values to control both EyeBlink and EyeOpen if (eyeCoefficients[0] > 0) { @@ -544,6 +560,10 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { } } +void DdeFaceTracker::setEyeClosingThreshold(float eyeClosingThreshold) { + _eyeClosingThreshold.set(eyeClosingThreshold); +} + static const int CALIBRATION_BILLBOARD_WIDTH = 240; static const int CALIBRATION_BILLBOARD_HEIGHT = 180; static const int CALIBRATION_BILLBOARD_TOP_MARGIN = 60; diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index 7019802603..8c30c5a4c3 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -50,6 +50,9 @@ public: float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } + float getEyeClosingThreshold() { return _eyeClosingThreshold.get(); } + void setEyeClosingThreshold(float eyeClosingThreshold); + public slots: void setEnabled(bool enabled); void calibrate(); @@ -89,8 +92,7 @@ private: int _rightBlinkIndex; int _leftEyeOpenIndex; int _rightEyeOpenIndex; - - // Brows + int _browDownLeftIndex; int _browDownRightIndex; int _browUpCenterIndex; @@ -114,6 +116,7 @@ private: float _filteredBrowUp; enum EyeState { + EYE_UNCONTROLLED, EYE_OPEN, EYE_CLOSING, EYE_CLOSED, @@ -123,6 +126,8 @@ private: float _lastEyeBlinks[2]; float _filteredEyeBlinks[2]; float _lastEyeCoefficients[2]; + Setting::Handle _eyeClosingThreshold; + QVector _coefficientAverages; bool _isCalibrating; diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index 4622ffd7ed..febd6f90bf 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -135,6 +136,10 @@ void PreferencesDialog::loadPreferences() { ui.pupilDilationSlider->setValue(myAvatar->getHead()->getPupilDilation() * ui.pupilDilationSlider->maximum()); + auto dde = DependencyManager::get(); + ui.ddeEyeClosingThresholdSlider->setValue(dde->getEyeClosingThreshold() * + ui.ddeEyeClosingThresholdSlider->maximum()); + auto faceshift = DependencyManager::get(); ui.faceshiftEyeDeflectionSider->setValue(faceshift->getEyeDeflection() * ui.faceshiftEyeDeflectionSider->maximum()); @@ -222,6 +227,10 @@ void PreferencesDialog::savePreferences() { qApp->setFieldOfView(ui.fieldOfViewSpin->value()); + auto dde = DependencyManager::get(); + dde->setEyeClosingThreshold(ui.ddeEyeClosingThresholdSlider->value() / + (float)ui.ddeEyeClosingThresholdSlider->maximum()); + auto faceshift = DependencyManager::get(); faceshift->setEyeDeflection(ui.faceshiftEyeDeflectionSider->value() / (float)ui.faceshiftEyeDeflectionSider->maximum()); diff --git a/interface/ui/preferencesDialog.ui b/interface/ui/preferencesDialog.ui index d26bdaee8a..e74b89075e 100644 --- a/interface/ui/preferencesDialog.ui +++ b/interface/ui/preferencesDialog.ui @@ -1256,7 +1256,7 @@ - Pupil dillation + Pupil dilation 0 @@ -1310,6 +1310,82 @@ + + + + 0 + + + 7 + + + 0 + + + 7 + + + + + + Arial + + + + Camera binary eyelid threshold + + + 0 + + + ddeEyeClosingThresholdSlider + + + + + + + + Arial + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + + 130 + 0 + + + + + Arial + + + + Qt::Horizontal + + + + +