diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 26892ea981..0227003cb1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1748,7 +1748,10 @@ void Application::setActiveFaceTracker() { DependencyManager::get()->setTCPEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift)); #endif #ifdef HAVE_DDE - DependencyManager::get()->setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression)); + bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression); + Menu::getInstance()->getActionForOption(MenuOption::DDEFiltering)->setVisible(isUsingDDE); + Menu::getInstance()->getActionForOption(MenuOption::ResetDDETracking)->setVisible(isUsingDDE); + DependencyManager::get()->setEnabled(isUsingDDE); #endif #ifdef HAVE_VISAGE DependencyManager::get()->updateEnabled(); diff --git a/interface/src/Application.h b/interface/src/Application.h index 833d9a6799..b5b2921844 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -210,7 +210,6 @@ public: bool getLastMouseMoveWasSimulated() const { return _lastMouseMoveWasSimulated; } FaceTracker* getActiveFaceTracker(); - void setActiveFaceTracker(); QSystemTrayIcon* getTrayIcon() { return _trayIcon; } ApplicationOverlay& getApplicationOverlay() { return _applicationOverlay; } @@ -385,6 +384,8 @@ public slots: void setVSyncEnabled(); void resetSensors(); + void setActiveFaceTracker(); + void aboutApp(); void showEditEntitiesHelp(); diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index bde0745c4d..8d6a1e3184 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -362,30 +362,32 @@ Menu::Menu() { QAction* noFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::NoFaceTracking, 0, true, - this, SLOT(setActiveFaceTracker())); + qApp, SLOT(setActiveFaceTracker())); faceTrackerGroup->addAction(noFaceTracker); #ifdef HAVE_FACESHIFT QAction* faceshiftFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::Faceshift, 0, false, - this, SLOT(setActiveFaceTracker())); + qApp, SLOT(setActiveFaceTracker())); faceTrackerGroup->addAction(faceshiftFaceTracker); #endif #ifdef HAVE_DDE QAction* ddeFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::DDEFaceRegression, 0, false, - this, SLOT(setActiveFaceTracker())); + qApp, SLOT(setActiveFaceTracker())); faceTrackerGroup->addAction(ddeFaceTracker); #endif #ifdef HAVE_VISAGE QAction* visageFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::Visage, 0, false, - this, SLOT(setActiveFaceTracker())); + qApp, SLOT(setActiveFaceTracker())); faceTrackerGroup->addAction(visageFaceTracker); #endif } #ifdef HAVE_DDE faceTrackingMenu->addSeparator(); + QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::DDEFiltering, 0, true); + ddeFiltering->setVisible(false); QAction* ddeFaceTrackerReset = addActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::ResetDDETracking, Qt::CTRL | Qt::Key_Apostrophe, DependencyManager::get().data(), SLOT(resetTracking())); @@ -987,11 +989,3 @@ void Menu::visibilityChanged(Discoverability::Mode discoverabilityMode) { qCDebug(interfaceapp) << "ERROR Menu::visibilityChanged() called with unrecognized value."; } } - -void Menu::setActiveFaceTracker() { -#ifdef HAVE_DDE - bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression); - Menu::getInstance()->getActionForOption(MenuOption::ResetDDETracking)->setVisible(isUsingDDE); -#endif - qApp->setActiveFaceTracker(); -} diff --git a/interface/src/Menu.h b/interface/src/Menu.h index f580b9204b..ea34664eaa 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -68,7 +68,6 @@ public slots: private slots: void setVisibility(); - void setActiveFaceTracker(); private: static Menu* _instance; @@ -137,6 +136,7 @@ namespace MenuOption { const QString CopyAddress = "Copy Address to Clipboard"; const QString CopyPath = "Copy Path to Clipboard"; const QString DDEFaceRegression = "DDE Face Regression"; + const QString DDEFiltering = "DDE Filtering"; const QString DecreaseAvatarSize = "Decrease Avatar Size"; const QString DeleteBookmark = "Delete Bookmark..."; const QString DisableActivityLogger = "Disable Activity Logger"; diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index c7a8b7d166..50601421c2 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -17,6 +17,8 @@ #include #include +#include + #include "DdeFaceTracker.h" #include "FaceshiftConstants.h" #include "InterfaceLogging.h" @@ -27,9 +29,9 @@ static const QHostAddress DDE_SERVER_ADDR("127.0.0.1"); static const quint16 DDE_SERVER_PORT = 64204; static const quint16 DDE_CONTROL_PORT = 64205; #if defined(Q_OS_WIN) -static const QString DDE_PROGRAM_PATH = QCoreApplication::applicationDirPath() + "/dde/dde.exe"; +static const QString DDE_PROGRAM_PATH = "/dde/dde.exe"; #elif defined(Q_OS_MAC) -static const QString DDE_PROGRAM_PATH = QCoreApplication::applicationDirPath() + "/dde.app/Contents/MacOS/dde"; +static const QString DDE_PROGRAM_PATH = "/dde.app/Contents/MacOS/dde"; #endif static const QStringList DDE_ARGUMENTS = QStringList() << "--udp=" + DDE_SERVER_ADDR.toString() + ":" + QString::number(DDE_SERVER_PORT) @@ -132,6 +134,8 @@ struct Packet { char name[MAX_NAME_SIZE + 1]; }; +const float STARTING_DDE_MESSAGE_TIME = 0.033f; + DdeFaceTracker::DdeFaceTracker() : DdeFaceTracker(QHostAddress::Any, DDE_SERVER_PORT, DDE_CONTROL_PORT) { @@ -157,11 +161,16 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui _mouthSmileLeftIndex(28), _mouthSmileRightIndex(29), _jawOpenIndex(21), - _previousTranslation(glm::vec3()), - _previousRotation(glm::quat()) + _lastMessageReceived(0), + _averageMessageTime(STARTING_DDE_MESSAGE_TIME), + _lastHeadTranslation(glm::vec3(0.0f)), + _filteredHeadTranslation(glm::vec3(0.0f)), + _lastLeftEyeBlink(0.0f), + _filteredLeftEyeBlink(0.0f), + _lastRightEyeBlink(0.0f), + _filteredRightEyeBlink(0.0f) { _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES); - _previousCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES); _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES); @@ -272,6 +281,8 @@ float DdeFaceTracker::getBlendshapeCoefficient(int index) const { void DdeFaceTracker::decodePacket(const QByteArray& buffer) { if(buffer.size() > MIN_PACKET_SIZE) { + bool isFiltering = Menu::getInstance()->isOptionChecked(MenuOption::DDEFiltering); + Packet packet; int bytesToCopy = glm::min((int)sizeof(packet), buffer.size()); memset(&packet.name, '\n', MAX_NAME_SIZE + 1); @@ -292,13 +303,36 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { translation -= _referenceTranslation; translation /= LEAN_DAMPING_FACTOR; translation.x *= -1; - _headTranslation = (translation + _previousTranslation) / 2.0f; - _previousTranslation = translation; + if (isFiltering) { + glm::vec3 linearVelocity = (translation - _lastHeadTranslation) / _averageMessageTime; + const float LINEAR_VELOCITY_FILTER_STRENGTH = 0.3f; + float velocityFilter = glm::clamp(1.0f - glm::length(linearVelocity) * + LINEAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); + _filteredHeadTranslation = velocityFilter * _filteredHeadTranslation + (1.0f - velocityFilter) * translation; + _lastHeadTranslation = translation; + _headTranslation = _filteredHeadTranslation; + } else { + _headTranslation = translation; + } // Compute relative rotation rotation = glm::inverse(_referenceRotation) * rotation; - _headRotation = (rotation + _previousRotation) / 2.0f; - _previousRotation = rotation; + if (isFiltering) { + glm::quat r = rotation * glm::inverse(_headRotation); + float theta = 2 * acos(r.w); + glm::vec3 angularVelocity; + if (theta > EPSILON) { + float rMag = glm::length(glm::vec3(r.x, r.y, r.z)); + angularVelocity = theta / _averageMessageTime * glm::vec3(r.x, r.y, r.z) / rMag; + } else { + angularVelocity = glm::vec3(0, 0, 0); + } + const float ANGULAR_VELOCITY_FILTER_STRENGTH = 0.3f; + _headRotation = safeMix(_headRotation, rotation, glm::clamp(glm::length(angularVelocity) * + ANGULAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f)); + } else { + _headRotation = rotation; + } // Translate DDE coefficients to Faceshift compatible coefficients for (int i = 0; i < NUM_EXPRESSIONS; i += 1) { @@ -307,8 +341,23 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { // Use EyeBlink values to control both EyeBlink and EyeOpen static const float RELAXED_EYE_VALUE = 0.1f; - float leftEye = (_coefficients[_leftBlinkIndex] + _previousCoefficients[_leftBlinkIndex]) / 2.0f; - float rightEye = (_coefficients[_rightBlinkIndex] + _previousCoefficients[_rightBlinkIndex]) / 2.0f; + float leftEye = _coefficients[_leftBlinkIndex]; + float rightEye = _coefficients[_rightBlinkIndex]; + if (isFiltering) { + const float BLINK_VELOCITY_FILTER_STRENGTH = 0.3f; + + float velocity = fabs(leftEye - _lastLeftEyeBlink) / _averageMessageTime; + float velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); + _filteredLeftEyeBlink = velocityFilter * leftEye + (1.0f - velocityFilter) * _filteredLeftEyeBlink; + _lastLeftEyeBlink = leftEye; + leftEye = _filteredLeftEyeBlink; + + velocity = fabs(rightEye - _lastRightEyeBlink) / _averageMessageTime; + velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); + _filteredRightEyeBlink = velocityFilter * rightEye + (1.0f - velocityFilter) * _filteredRightEyeBlink; + _lastRightEyeBlink = rightEye; + rightEye = _filteredRightEyeBlink; + } if (leftEye > RELAXED_EYE_VALUE) { _coefficients[_leftBlinkIndex] = leftEye - RELAXED_EYE_VALUE; _coefficients[_leftEyeOpenIndex] = 0.0f; @@ -343,10 +392,18 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { // Scale all coefficients for (int i = 0; i < NUM_EXPRESSIONS; i += 1) { _blendshapeCoefficients[i] - = glm::clamp(DDE_COEFFICIENT_SCALES[i] * (_coefficients[i] + _previousCoefficients[i]) / 2.0f, 0.0f, 1.0f); - _previousCoefficients[i] = _coefficients[i]; + = glm::clamp(DDE_COEFFICIENT_SCALES[i] * _coefficients[i], 0.0f, 1.0f); } + // Calculate average frame time + const float FRAME_AVERAGING_FACTOR = 0.99f; + quint64 usecsNow = usecTimestampNow(); + if (_lastMessageReceived != 0) { + _averageMessageTime = FRAME_AVERAGING_FACTOR * _averageMessageTime + + (1.0f - FRAME_AVERAGING_FACTOR) * (float)(usecsNow - _lastMessageReceived) / 1000000.0f; + } + _lastMessageReceived = usecsNow; + } else { qCDebug(interfaceapp) << "[Error] DDE Face Tracker Decode Error"; } diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index 490020e511..aa3e9f3b81 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -101,10 +101,14 @@ private: QVector _coefficients; - // Previous values for simple smoothing - glm::vec3 _previousTranslation; - glm::quat _previousRotation; - QVector _previousCoefficients; + quint64 _lastMessageReceived; + float _averageMessageTime; + glm::vec3 _lastHeadTranslation; + glm::vec3 _filteredHeadTranslation; + float _lastLeftEyeBlink; + float _filteredLeftEyeBlink; + float _lastRightEyeBlink; + float _filteredRightEyeBlink; }; #endif // hifi_DdeFaceTracker_h \ No newline at end of file