Merge pull request #4786 from ctrlaltdavid/20530

CR for #20530 - Add DDE calibration

interface/src/Application.cpp

@@ -1946,6 +1946,7 @@ void Application::setActiveFaceTracker() {
     bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::UseCamera);
     Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE);
     Menu::getInstance()->getActionForOption(MenuOption::VelocityFilter)->setVisible(isUsingDDE);
+    Menu::getInstance()->getActionForOption(MenuOption::CalibrateCamera)->setVisible(isUsingDDE);
     auto ddeTracker = DependencyManager::get<DdeFaceTracker>();
     ddeTracker->setIsMuted(isMuted);
     ddeTracker->setEnabled(isUsingDDE && !isMuted);

interface/src/Menu.cpp

@@ -392,6 +392,9 @@ Menu::Menu() {
     useAudioForMouth->setVisible(false);
     QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::VelocityFilter, 0, true);
     ddeFiltering->setVisible(false);
+    QAction* ddeCalibrate = addActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::CalibrateCamera, 0,
+        DependencyManager::get<DdeFaceTracker>().data(), SLOT(calibrate()));
+    ddeCalibrate->setVisible(false);
 #endif
 #if defined(HAVE_FACESHIFT) || defined(HAVE_DDE)
     faceTrackingMenu->addSeparator();

interface/src/Menu.h

@@ -154,6 +154,7 @@ namespace MenuOption {
     const QString Bookmarks = "Bookmarks";
     const QString CascadedShadows = "Cascaded";
     const QString CachesSize = "RAM Caches Size";
+    const QString CalibrateCamera = "Calibrate Camera";
     const QString Chat = "Chat...";
     const QString Collisions = "Collisions";
     const QString Console = "Console...";

interface/src/devices/DdeFaceTracker.cpp

@@ -20,6 +20,7 @@
 #include <GLMHelpers.h>
 #include <NumericalConstants.h>
 
+#include "Application.h"
 #include "DdeFaceTracker.h"
 #include "FaceshiftConstants.h"
 #include "InterfaceLogging.h"
@@ -86,16 +87,16 @@ static const float DDE_COEFFICIENT_SCALES[] = {
     3.0f, // BrowsU_L
     3.0f, // BrowsU_R
     1.0f, // JawFwd
-    1.5f, // JawLeft
+    2.0f, // JawLeft
     1.8f, // JawOpen
     1.0f, // JawChew
-    1.5f, // JawRight
+    2.0f, // JawRight
     1.5f, // MouthLeft
     1.5f, // MouthRight
     1.5f, // MouthFrown_L
     1.5f, // MouthFrown_R
-    1.5f, // MouthSmile_L
+    2.5f, // MouthSmile_L
-    1.5f, // MouthSmile_R
+    2.5f, // MouthSmile_R
     1.0f, // MouthDimple_L
     1.0f, // MouthDimple_R
     1.0f, // LipsStretch_L
@@ -106,8 +107,8 @@ static const float DDE_COEFFICIENT_SCALES[] = {
     1.0f, // LipsLowerDown
     1.0f, // LipsUpperOpen
     1.0f, // LipsLowerOpen
-    2.5f, // LipsFunnel
+    1.5f, // LipsFunnel
-    2.0f, // LipsPucker
+    2.5f, // LipsPucker
     1.5f, // ChinLowerRaise
     1.5f, // ChinUpperRaise
     1.0f, // Sneer
@@ -135,7 +136,9 @@ struct Packet {
     char name[MAX_NAME_SIZE + 1];
 };
 
-const float STARTING_DDE_MESSAGE_TIME = 0.033f;
+static const float STARTING_DDE_MESSAGE_TIME = 0.033f;
+
+static const int CALIBRATION_SAMPLES = 150;
 
 #ifdef WIN32
 //  warning C4351: new behavior: elements of array 'DdeFaceTracker::_lastEyeBlinks' will be default initialized 
@@ -178,11 +181,18 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
     _filteredBrowUp(0.0f),
     _lastEyeBlinks(),
     _filteredEyeBlinks(),
-    _lastEyeCoefficients()
+    _lastEyeCoefficients(),
+    _isCalibrating(false),
+    _calibrationValues(),
+    _calibrationCount(0),
+    _calibrationBillboard(NULL),
+    _calibrationBillboardID(0),
+    _calibrationMessage(QString())
 {
     _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES);
-
     _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES);
+    _coefficientAverages.resize(NUM_FACESHIFT_BLENDSHAPES);
+    _calibrationValues.resize(NUM_FACESHIFT_BLENDSHAPES);
 
     _eyeStates[0] = EYE_OPEN;
     _eyeStates[1] = EYE_OPEN;
@@ -195,6 +205,10 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
 
 DdeFaceTracker::~DdeFaceTracker() {
     setEnabled(false);
+
+    if (_isCalibrating) {
+        cancelCalibration();
+    }
 }
 
 #ifdef WIN32
@@ -212,6 +226,12 @@ void DdeFaceTracker::setEnabled(bool enabled) {
         return;
     }
 #ifdef HAVE_DDE
+
+    if (_isCalibrating) {
+        cancelCalibration();
+    }
+
+
     // isOpen() does not work as one might expect on QUdpSocket; don't test isOpen() before closing socket.
     _udpSocket.close();
     if (enabled) {
@@ -382,10 +402,18 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
             _coefficients[DDE_TO_FACESHIFT_MAPPING[i]] = packet.expressions[i];
         }
 
+        // Calibration
+        if (_isCalibrating) {
+            addCalibrationDatum();
+        }
+        for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) {
+            _coefficients[i] -= _coefficientAverages[i];
+        }
+
         // Use BrowsU_C to control both brows' up and down
         float browUp = _coefficients[_browUpCenterIndex];
         if (isFiltering) {
-            const float BROW_VELOCITY_FILTER_STRENGTH = 0.75f;
+            const float BROW_VELOCITY_FILTER_STRENGTH = 0.5f;
             float velocity = fabs(browUp - _lastBrowUp) / _averageMessageTime;
             float velocityFilter = glm::clamp(velocity * BROW_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);
             _filteredBrowUp = velocityFilter * browUp + (1.0f - velocityFilter) * _filteredBrowUp;
@@ -399,11 +427,11 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
         _coefficients[_browDownRightIndex] = -browUp;
 
         // Offset jaw open coefficient
-        static const float JAW_OPEN_THRESHOLD = 0.16f;
+        static const float JAW_OPEN_THRESHOLD = 0.1f;
         _coefficients[_jawOpenIndex] = _coefficients[_jawOpenIndex] - JAW_OPEN_THRESHOLD;
 
         // Offset smile coefficients
-        static const float SMILE_THRESHOLD = 0.18f;
+        static const float SMILE_THRESHOLD = 0.5f;
         _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD;
         _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD;
 
@@ -430,7 +458,7 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
 
             // Change to closing or opening states
             const float EYE_CONTROL_HYSTERISIS = 0.25f;
-            const float EYE_CLOSING_THRESHOLD = 0.95f;
+            const float EYE_CLOSING_THRESHOLD = 0.8f;
             const float EYE_OPENING_THRESHOLD = EYE_CONTROL_THRESHOLD - EYE_CONTROL_HYSTERISIS;
             if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > EYE_CLOSING_THRESHOLD) {
                 _eyeStates[i] = EYE_CLOSING;
@@ -510,4 +538,81 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
     } else {
         qCWarning(interfaceapp) << "DDE Face Tracker: Decode error";
     }
+
+    if (_isCalibrating && _calibrationCount > CALIBRATION_SAMPLES) {
+        finishCalibration();
+    }
+}
+
+static const int CALIBRATION_BILLBOARD_WIDTH = 240;
+static const int CALIBRATION_BILLBOARD_HEIGHT = 180;
+static const int CALIBRATION_BILLBOARD_TOP_MARGIN = 60;
+static const int CALIBRATION_BILLBOARD_LEFT_MARGIN = 30;
+static const int CALIBRATION_BILLBOARD_FONT_SIZE = 16;
+static const float CALIBRATION_BILLBOARD_ALPHA = 0.5f;
+static QString CALIBRATION_INSTRUCTION_MESSAGE = "Hold still to calibrate";
+
+void DdeFaceTracker::calibrate() {
+    if (!_isCalibrating) {
+        qCDebug(interfaceapp) << "DDE Face Tracker: Calibration started";
+
+        _isCalibrating = true;
+        _calibrationCount = 0;
+        _calibrationMessage = CALIBRATION_INSTRUCTION_MESSAGE + "\n\n";
+
+        _calibrationBillboard = new TextOverlay();
+        _calibrationBillboard->setTopMargin(CALIBRATION_BILLBOARD_TOP_MARGIN);
+        _calibrationBillboard->setLeftMargin(CALIBRATION_BILLBOARD_LEFT_MARGIN);
+        _calibrationBillboard->setFontSize(CALIBRATION_BILLBOARD_FONT_SIZE);
+        _calibrationBillboard->setText(CALIBRATION_INSTRUCTION_MESSAGE);
+        _calibrationBillboard->setAlpha(CALIBRATION_BILLBOARD_ALPHA);
+        glm::vec2 viewport = qApp->getCanvasSize();
+        _calibrationBillboard->setX((viewport.x - CALIBRATION_BILLBOARD_WIDTH) / 2);
+        _calibrationBillboard->setY((viewport.y - CALIBRATION_BILLBOARD_HEIGHT) / 2);
+        _calibrationBillboard->setWidth(CALIBRATION_BILLBOARD_WIDTH);
+        _calibrationBillboard->setHeight(CALIBRATION_BILLBOARD_HEIGHT);
+        _calibrationBillboardID = qApp->getOverlays().addOverlay(_calibrationBillboard);
+
+        for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) {
+            _calibrationValues[i] = 0.0f;
+        }
+    }
+}
+
+void DdeFaceTracker::addCalibrationDatum() {
+    const int LARGE_TICK_INTERVAL = 30;
+    const int SMALL_TICK_INTERVAL = 6;
+    int samplesLeft = CALIBRATION_SAMPLES - _calibrationCount;
+    if (samplesLeft % LARGE_TICK_INTERVAL == 0) {
+        _calibrationMessage += QString::number(samplesLeft / LARGE_TICK_INTERVAL);
+        _calibrationBillboard->setText(_calibrationMessage);
+    } else if (samplesLeft % SMALL_TICK_INTERVAL == 0) {
+        _calibrationMessage += ".";
+        _calibrationBillboard->setText(_calibrationMessage);
+    }
+
+    for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) {
+        _calibrationValues[i] += _coefficients[i];
+    }
+
+    _calibrationCount += 1;
+}
+
+void DdeFaceTracker::cancelCalibration() {
+    qApp->getOverlays().deleteOverlay(_calibrationBillboardID);
+    _calibrationBillboard = NULL;
+    _isCalibrating = false;
+    qCDebug(interfaceapp) << "DDE Face Tracker: Calibration cancelled";
+}
+
+void DdeFaceTracker::finishCalibration() {
+    qApp->getOverlays().deleteOverlay(_calibrationBillboardID);
+    _calibrationBillboard = NULL;
+    _isCalibrating = false;
+
+    for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) {
+        _coefficientAverages[i] = _calibrationValues[i] / (float)CALIBRATION_SAMPLES;
+    }
+
+    qCDebug(interfaceapp) << "DDE Face Tracker: Calibration finished";
 }

interface/src/devices/DdeFaceTracker.h

@@ -20,6 +20,7 @@
 #include <QUdpSocket>
 
 #include <DependencyManager.h>
+#include <ui/overlays/TextOverlay.h>
 
 #include "FaceTracker.h"
 
@@ -51,6 +52,7 @@ public:
 
 public slots:
     void setEnabled(bool enabled);
+    void calibrate();
 
 private slots:
     void processFinished(int exitCode, QProcess::ExitStatus exitStatus);
@@ -121,6 +123,17 @@ private:
     float _lastEyeBlinks[2];
     float _filteredEyeBlinks[2];
     float _lastEyeCoefficients[2];
+    QVector<float> _coefficientAverages;
+
+    bool _isCalibrating;
+    int _calibrationCount;
+    QVector<float> _calibrationValues;
+    TextOverlay* _calibrationBillboard;
+    int _calibrationBillboardID;
+    QString _calibrationMessage;
+    void addCalibrationDatum();
+    void cancelCalibration();
+    void finishCalibration();
 };
 
 #endif // hifi_DdeFaceTracker_h