From 667ad9b0cb6c9db029251f83987de94d36c3863b Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 11 May 2015 15:48:51 -0700 Subject: [PATCH] Add menu option to enable/disable binary webcam eyelid control --- interface/src/Application.cpp | 1 + interface/src/Menu.cpp | 2 + interface/src/Menu.h | 1 + interface/src/devices/DdeFaceTracker.cpp | 109 +++++++++++++---------- interface/src/devices/DdeFaceTracker.h | 1 + 5 files changed, 67 insertions(+), 47 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 088b2ddf96..ad7e788d5f 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1947,6 +1947,7 @@ void Application::setActiveFaceTracker() { #endif #ifdef HAVE_DDE bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::UseCamera); + Menu::getInstance()->getActionForOption(MenuOption::BinaryEyelidControl)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::VelocityFilter)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setVisible(isUsingDDE); diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 61213e7334..e1587684cf 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -388,6 +388,8 @@ Menu::Menu() { } #ifdef HAVE_DDE faceTrackingMenu->addSeparator(); + QAction* binaryEyelidControl = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::BinaryEyelidControl, 0, true); + binaryEyelidControl->setVisible(false); QAction* useAudioForMouth = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseAudioForMouth, 0, true); useAudioForMouth->setVisible(false); QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::VelocityFilter, 0, true); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 96c8fedf0d..435df47487 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -149,6 +149,7 @@ namespace MenuOption { const QString AudioStatsShowInjectedStreams = "Audio Stats Show Injected Streams"; const QString AvatarReceiveStats = "Show Receive Stats"; const QString BandwidthDetails = "Bandwidth Details"; + const QString BinaryEyelidControl = "Binary Eyelid Control"; const QString BlueSpeechSphere = "Blue Sphere While Speaking"; const QString BookmarkLocation = "Bookmark Location"; const QString Bookmarks = "Bookmarks"; diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index 6ed253c1ec..4141f9fb82 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -194,8 +194,8 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui _coefficientAverages.resize(NUM_FACESHIFT_BLENDSHAPES); _calibrationValues.resize(NUM_FACESHIFT_BLENDSHAPES); - _eyeStates[0] = EYE_OPEN; - _eyeStates[1] = EYE_OPEN; + _eyeStates[0] = EYE_UNCONTROLLED; + _eyeStates[1] = EYE_UNCONTROLLED; connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError))); @@ -450,57 +450,72 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { // Finesse EyeBlink values float eyeCoefficients[2]; - for (int i = 0; i < 2; i++) { - // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen - // -ve values control EyeOpen; +ve values control EyeBlink - static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value - eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD); + if (Menu::getInstance()->isOptionChecked(MenuOption::BinaryEyelidControl)) { + if (_eyeStates[0] == EYE_UNCONTROLLED) { + _eyeStates[0] = EYE_OPEN; + _eyeStates[1] = EYE_OPEN; + } - // Change to closing or opening states - const float EYE_CONTROL_HYSTERISIS = 0.25f; - const float EYE_CLOSING_THRESHOLD = 0.8f; - const float EYE_OPENING_THRESHOLD = EYE_CONTROL_THRESHOLD - EYE_CONTROL_HYSTERISIS; - if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > EYE_CLOSING_THRESHOLD) { - _eyeStates[i] = EYE_CLOSING; - } else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING) + for (int i = 0; i < 2; i++) { + // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen + // -ve values control EyeOpen; +ve values control EyeBlink + static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value + eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD); + + // Change to closing or opening states + const float EYE_CONTROL_HYSTERISIS = 0.25f; + const float EYE_CLOSING_THRESHOLD = 0.8f; + const float EYE_OPENING_THRESHOLD = EYE_CONTROL_THRESHOLD - EYE_CONTROL_HYSTERISIS; + if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > EYE_CLOSING_THRESHOLD) { + _eyeStates[i] = EYE_CLOSING; + } else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING) && eyeCoefficients[i] < EYE_OPENING_THRESHOLD) { - _eyeStates[i] = EYE_OPENING; + _eyeStates[i] = EYE_OPENING; + } + + const float EYELID_MOVEMENT_RATE = 10.0f; // units/second + const float EYE_OPEN_SCALE = 0.2f; + if (_eyeStates[i] == EYE_CLOSING) { + // Close eyelid until it's fully closed + float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime; + if (closingValue >= 1.0) { + _eyeStates[i] = EYE_CLOSED; + eyeCoefficients[i] = 1.0; + } else { + eyeCoefficients[i] = closingValue; + } + } else if (_eyeStates[i] == EYE_OPENING) { + // Open eyelid until it meets the current adjusted value + float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime; + if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) { + _eyeStates[i] = EYE_OPEN; + eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; + } else { + eyeCoefficients[i] = openingValue; + } + } else if (_eyeStates[i] == EYE_OPEN) { + // Reduce eyelid movement + eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; + } else if (_eyeStates[i] == EYE_CLOSED) { + // Keep eyelid fully closed + eyeCoefficients[i] = 1.0; + } } - const float EYELID_MOVEMENT_RATE = 10.0f; // units/second - const float EYE_OPEN_SCALE = 0.2f; - if (_eyeStates[i] == EYE_CLOSING) { - // Close eyelid until it's fully closed - float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime; - if (closingValue >= 1.0) { - _eyeStates[i] = EYE_CLOSED; - eyeCoefficients[i] = 1.0; - } else { - eyeCoefficients[i] = closingValue; - } - } else if (_eyeStates[i] == EYE_OPENING) { - // Open eyelid until it meets the current adjusted value - float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime; - if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) { - _eyeStates[i] = EYE_OPEN; - eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else { - eyeCoefficients[i] = openingValue; - } - } else if (_eyeStates[i] == EYE_OPEN) { - // Reduce eyelid movement - eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; - } else if (_eyeStates[i] == EYE_CLOSED) { - // Keep eyelid fully closed - eyeCoefficients[i] = 1.0; + if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) { + // Couple eyelids + eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f; } + + _lastEyeCoefficients[0] = eyeCoefficients[0]; + _lastEyeCoefficients[1] = eyeCoefficients[1]; + } else { + _eyeStates[0] = EYE_UNCONTROLLED; + _eyeStates[1] = EYE_UNCONTROLLED; + + eyeCoefficients[0] = _filteredEyeBlinks[0]; + eyeCoefficients[1] = _filteredEyeBlinks[1]; } - if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) { - // Couple eyelids - eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f; - } - _lastEyeCoefficients[0] = eyeCoefficients[0]; - _lastEyeCoefficients[1] = eyeCoefficients[1]; // Use EyeBlink values to control both EyeBlink and EyeOpen if (eyeCoefficients[0] > 0) { diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index 7019802603..f627464c6f 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -114,6 +114,7 @@ private: float _filteredBrowUp; enum EyeState { + EYE_UNCONTROLLED, EYE_OPEN, EYE_CLOSING, EYE_CLOSED,