Merge pull request #4658 from ctrlaltdavid/dde-filtering

DO NOT MERGE - DDE filtering

interface/src/Application.cpp

@@ -1748,7 +1748,10 @@ void Application::setActiveFaceTracker() {
     DependencyManager::get<Faceshift>()->setTCPEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift));
 #endif
 #ifdef HAVE_DDE
-    DependencyManager::get<DdeFaceTracker>()->setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression));
+    bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression);
+    Menu::getInstance()->getActionForOption(MenuOption::DDEFiltering)->setVisible(isUsingDDE);
+    Menu::getInstance()->getActionForOption(MenuOption::ResetDDETracking)->setVisible(isUsingDDE);
+    DependencyManager::get<DdeFaceTracker>()->setEnabled(isUsingDDE);
 #endif
 #ifdef HAVE_VISAGE
     DependencyManager::get<Visage>()->updateEnabled();

interface/src/Application.h

@@ -210,7 +210,6 @@ public:
     bool getLastMouseMoveWasSimulated() const { return _lastMouseMoveWasSimulated; }
     
     FaceTracker* getActiveFaceTracker();
-    void setActiveFaceTracker();
 
     QSystemTrayIcon* getTrayIcon() { return _trayIcon; }
     ApplicationOverlay& getApplicationOverlay() { return _applicationOverlay; }
@@ -385,6 +384,8 @@ public slots:
     void setVSyncEnabled();
 
     void resetSensors();
+    void setActiveFaceTracker();
+
     void aboutApp();
     void showEditEntitiesHelp();
     

interface/src/Menu.cpp

@@ -362,30 +362,32 @@ Menu::Menu() {
 
         QAction* noFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::NoFaceTracking,
             0, true,
-            this, SLOT(setActiveFaceTracker()));
+            qApp, SLOT(setActiveFaceTracker()));
         faceTrackerGroup->addAction(noFaceTracker);
 
 #ifdef HAVE_FACESHIFT
         QAction* faceshiftFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::Faceshift,
             0, false,
-            this, SLOT(setActiveFaceTracker()));
+            qApp, SLOT(setActiveFaceTracker()));
         faceTrackerGroup->addAction(faceshiftFaceTracker);
 #endif
 #ifdef HAVE_DDE
         QAction* ddeFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::DDEFaceRegression, 
             0, false,
-            this, SLOT(setActiveFaceTracker()));
+            qApp, SLOT(setActiveFaceTracker()));
         faceTrackerGroup->addAction(ddeFaceTracker);
 #endif
 #ifdef HAVE_VISAGE
         QAction* visageFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::Visage, 
             0, false,
-            this, SLOT(setActiveFaceTracker()));
+            qApp, SLOT(setActiveFaceTracker()));
         faceTrackerGroup->addAction(visageFaceTracker);
 #endif
     }
 #ifdef HAVE_DDE
     faceTrackingMenu->addSeparator();
+    QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::DDEFiltering, 0, true);
+    ddeFiltering->setVisible(false);
     QAction* ddeFaceTrackerReset = addActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::ResetDDETracking, 
         Qt::CTRL | Qt::Key_Apostrophe,
         DependencyManager::get<DdeFaceTracker>().data(), SLOT(resetTracking()));
@@ -987,11 +989,3 @@ void Menu::visibilityChanged(Discoverability::Mode discoverabilityMode) {
         qCDebug(interfaceapp) << "ERROR Menu::visibilityChanged() called with unrecognized value.";
     }
 }
-
-void Menu::setActiveFaceTracker() {
-#ifdef HAVE_DDE
-    bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression);
-    Menu::getInstance()->getActionForOption(MenuOption::ResetDDETracking)->setVisible(isUsingDDE);
-#endif
-    qApp->setActiveFaceTracker();
-}

interface/src/Menu.h

@@ -68,7 +68,6 @@ public slots:
     
 private slots:
     void setVisibility();
-    void setActiveFaceTracker();
 
 private:
     static Menu* _instance;
@@ -137,6 +136,7 @@ namespace MenuOption {
     const QString CopyAddress = "Copy Address to Clipboard";
     const QString CopyPath = "Copy Path to Clipboard";
     const QString DDEFaceRegression = "DDE Face Regression";
+    const QString DDEFiltering = "DDE Filtering";
     const QString DecreaseAvatarSize = "Decrease Avatar Size";
     const QString DeleteBookmark = "Delete Bookmark...";
     const QString DisableActivityLogger = "Disable Activity Logger";

interface/src/devices/DdeFaceTracker.cpp

@@ -17,6 +17,8 @@
 #include <QJsonObject>
 #include <QElapsedTimer>
 
+#include <GLMHelpers.h>
+
 #include "DdeFaceTracker.h"
 #include "FaceshiftConstants.h"
 #include "InterfaceLogging.h"
@@ -27,9 +29,9 @@ static const QHostAddress DDE_SERVER_ADDR("127.0.0.1");
 static const quint16 DDE_SERVER_PORT = 64204;
 static const quint16 DDE_CONTROL_PORT = 64205;
 #if defined(Q_OS_WIN)
-static const QString DDE_PROGRAM_PATH = QCoreApplication::applicationDirPath() + "/dde/dde.exe";
+static const QString DDE_PROGRAM_PATH = "/dde/dde.exe";
 #elif defined(Q_OS_MAC)
-static const QString DDE_PROGRAM_PATH = QCoreApplication::applicationDirPath() + "/dde.app/Contents/MacOS/dde";
+static const QString DDE_PROGRAM_PATH = "/dde.app/Contents/MacOS/dde";
 #endif
 static const QStringList DDE_ARGUMENTS = QStringList() 
     << "--udp=" + DDE_SERVER_ADDR.toString() + ":" + QString::number(DDE_SERVER_PORT) 
@@ -132,6 +134,8 @@ struct Packet {
     char name[MAX_NAME_SIZE + 1];
 };
 
+const float STARTING_DDE_MESSAGE_TIME = 0.033f;
+
 DdeFaceTracker::DdeFaceTracker() :
     DdeFaceTracker(QHostAddress::Any, DDE_SERVER_PORT, DDE_CONTROL_PORT)
 {
@@ -157,11 +161,16 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
     _mouthSmileLeftIndex(28),
     _mouthSmileRightIndex(29),
     _jawOpenIndex(21),
-    _previousTranslation(glm::vec3()),
+    _lastMessageReceived(0),
-    _previousRotation(glm::quat())
+    _averageMessageTime(STARTING_DDE_MESSAGE_TIME),
+    _lastHeadTranslation(glm::vec3(0.0f)),
+    _filteredHeadTranslation(glm::vec3(0.0f)),
+    _lastLeftEyeBlink(0.0f),
+    _filteredLeftEyeBlink(0.0f),
+    _lastRightEyeBlink(0.0f),
+    _filteredRightEyeBlink(0.0f)
 {
     _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES);
-    _previousCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES);
 
     _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES);
     
@@ -272,6 +281,8 @@ float DdeFaceTracker::getBlendshapeCoefficient(int index) const {
 
 void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
     if(buffer.size() > MIN_PACKET_SIZE) {
+        bool isFiltering = Menu::getInstance()->isOptionChecked(MenuOption::DDEFiltering);
+
         Packet packet;
         int bytesToCopy = glm::min((int)sizeof(packet), buffer.size());
         memset(&packet.name, '\n', MAX_NAME_SIZE + 1);
@@ -292,13 +303,36 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
         translation -= _referenceTranslation;
         translation /= LEAN_DAMPING_FACTOR;
         translation.x *= -1;
-        _headTranslation = (translation + _previousTranslation) / 2.0f;
+        if (isFiltering) {
-        _previousTranslation = translation;
+            glm::vec3 linearVelocity = (translation - _lastHeadTranslation) / _averageMessageTime;
+            const float LINEAR_VELOCITY_FILTER_STRENGTH = 0.3f;
+            float velocityFilter = glm::clamp(1.0f - glm::length(linearVelocity) *
+                LINEAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);
+            _filteredHeadTranslation = velocityFilter * _filteredHeadTranslation + (1.0f - velocityFilter) * translation;
+            _lastHeadTranslation = translation;
+            _headTranslation = _filteredHeadTranslation;
+        } else {
+            _headTranslation = translation;
+        }
 
         // Compute relative rotation
         rotation = glm::inverse(_referenceRotation) * rotation;
-        _headRotation = (rotation + _previousRotation) / 2.0f;
+        if (isFiltering) {
-        _previousRotation = rotation;
+            glm::quat r = rotation * glm::inverse(_headRotation);
+            float theta = 2 * acos(r.w);
+            glm::vec3 angularVelocity;
+            if (theta > EPSILON) {
+                float rMag = glm::length(glm::vec3(r.x, r.y, r.z));
+                angularVelocity = theta / _averageMessageTime * glm::vec3(r.x, r.y, r.z) / rMag;
+            } else {
+                angularVelocity = glm::vec3(0, 0, 0);
+            }
+            const float ANGULAR_VELOCITY_FILTER_STRENGTH = 0.3f;
+            _headRotation = safeMix(_headRotation, rotation, glm::clamp(glm::length(angularVelocity) *
+                ANGULAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f));
+        } else {
+            _headRotation = rotation;
+        }
 
         // Translate DDE coefficients to Faceshift compatible coefficients
         for (int i = 0; i < NUM_EXPRESSIONS; i += 1) {
@@ -307,8 +341,23 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
 
         // Use EyeBlink values to control both EyeBlink and EyeOpen
         static const float RELAXED_EYE_VALUE = 0.1f;
-        float leftEye = (_coefficients[_leftBlinkIndex] + _previousCoefficients[_leftBlinkIndex]) / 2.0f;
+        float leftEye = _coefficients[_leftBlinkIndex];
-        float rightEye = (_coefficients[_rightBlinkIndex] + _previousCoefficients[_rightBlinkIndex]) / 2.0f;
+        float rightEye = _coefficients[_rightBlinkIndex];
+        if (isFiltering) {
+            const float BLINK_VELOCITY_FILTER_STRENGTH = 0.3f;
+
+            float velocity = fabs(leftEye - _lastLeftEyeBlink) / _averageMessageTime;
+            float velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);
+            _filteredLeftEyeBlink = velocityFilter * leftEye + (1.0f - velocityFilter) * _filteredLeftEyeBlink;
+            _lastLeftEyeBlink = leftEye;
+            leftEye = _filteredLeftEyeBlink;
+
+            velocity = fabs(rightEye - _lastRightEyeBlink) / _averageMessageTime;
+            velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);
+            _filteredRightEyeBlink = velocityFilter * rightEye + (1.0f - velocityFilter) * _filteredRightEyeBlink;
+            _lastRightEyeBlink = rightEye;
+            rightEye = _filteredRightEyeBlink;
+        }
         if (leftEye > RELAXED_EYE_VALUE) {
             _coefficients[_leftBlinkIndex] = leftEye - RELAXED_EYE_VALUE;
             _coefficients[_leftEyeOpenIndex] = 0.0f;
@@ -343,10 +392,18 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
         // Scale all coefficients
         for (int i = 0; i < NUM_EXPRESSIONS; i += 1) {
             _blendshapeCoefficients[i]
-                = glm::clamp(DDE_COEFFICIENT_SCALES[i] * (_coefficients[i] + _previousCoefficients[i]) / 2.0f, 0.0f, 1.0f);
+                = glm::clamp(DDE_COEFFICIENT_SCALES[i] * _coefficients[i], 0.0f, 1.0f);
-            _previousCoefficients[i] = _coefficients[i];
         }
 
+        // Calculate average frame time
+        const float FRAME_AVERAGING_FACTOR = 0.99f;
+        quint64 usecsNow = usecTimestampNow();
+        if (_lastMessageReceived != 0) {
+            _averageMessageTime = FRAME_AVERAGING_FACTOR * _averageMessageTime 
+                + (1.0f - FRAME_AVERAGING_FACTOR) * (float)(usecsNow - _lastMessageReceived) / 1000000.0f;
+        }
+        _lastMessageReceived = usecsNow;
+    
     } else {
         qCDebug(interfaceapp) << "[Error] DDE Face Tracker Decode Error";
     }

interface/src/devices/DdeFaceTracker.h

@@ -101,10 +101,14 @@ private:
 
     QVector<float> _coefficients;
 
-    // Previous values for simple smoothing
+    quint64 _lastMessageReceived;
-    glm::vec3 _previousTranslation;
+    float _averageMessageTime;
-    glm::quat _previousRotation;
+    glm::vec3 _lastHeadTranslation;
-    QVector<float> _previousCoefficients;
+    glm::vec3 _filteredHeadTranslation;
+    float _lastLeftEyeBlink;
+    float _filteredLeftEyeBlink;
+    float _lastRightEyeBlink;
+    float _filteredRightEyeBlink;
 };
 
 #endif // hifi_DdeFaceTracker_h