From 5c7913068e6aa13bc8e937ac60c252c1fbd8a6d9 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 16:21:08 -0700 Subject: [PATCH 1/5] Support for UDP input from Faceshift. --- interface/src/devices/Faceshift.cpp | 35 ++++++++++++++++++++--------- interface/src/devices/Faceshift.h | 7 ++++-- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 991a406cc7..66ba958bbd 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -8,13 +8,18 @@ #include +#include + #include "Faceshift.h" using namespace fs; using namespace std; +const quint16 FACESHIFT_PORT = 33433; + Faceshift::Faceshift() : _enabled(false), + _lastMessageReceived(0), _eyeGazeLeftPitch(0.0f), _eyeGazeLeftYaw(0.0f), _eyeGazeRightPitch(0.0f), @@ -32,9 +37,19 @@ Faceshift::Faceshift() : _estimatedEyePitch(0.0f), _estimatedEyeYaw(0.0f) { - connect(&_socket, SIGNAL(connected()), SLOT(noteConnected())); - connect(&_socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); - connect(&_socket, SIGNAL(readyRead()), SLOT(readFromSocket())); + connect(&_tcpSocket, SIGNAL(connected()), SLOT(noteConnected())); + connect(&_tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); + connect(&_tcpSocket, SIGNAL(readyRead()), SLOT(readFromSocket())); + + connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readFromSocket())); + + _udpSocket.bind(FACESHIFT_PORT); +} + +bool Faceshift::isActive() const { + const uint64_t ACTIVE_TIMEOUT_USECS = 1000000; + return (_tcpSocket.state() == QAbstractSocket::ConnectedState || + (usecTimestampNow() - _lastMessageReceived) < ACTIVE_TIMEOUT_USECS) && _tracking; } void Faceshift::update() { @@ -51,7 +66,7 @@ void Faceshift::update() { } void Faceshift::reset() { - if (isActive()) { + if (_tcpSocket.state() == QAbstractSocket::ConnectedState) { string message; fsBinaryStream::encode_message(message, fsMsgCalibrateNeutral()); send(message); @@ -63,7 +78,7 @@ void Faceshift::setEnabled(bool enabled) { connectSocket(); } else { - _socket.disconnectFromHost(); + _tcpSocket.disconnectFromHost(); } } @@ -71,8 +86,7 @@ void Faceshift::connectSocket() { if (_enabled) { qDebug("Faceshift: Connecting...\n"); - const quint16 FACESHIFT_PORT = 33433; - _socket.connectToHost("localhost", FACESHIFT_PORT); + _tcpSocket.connectToHost("localhost", FACESHIFT_PORT); _tracking = false; } } @@ -87,7 +101,7 @@ void Faceshift::noteConnected() { } void Faceshift::noteError(QAbstractSocket::SocketError error) { - qDebug() << "Faceshift: " << _socket.errorString() << "\n"; + qDebug() << "Faceshift: " << _tcpSocket.errorString() << "\n"; // reconnect after a delay if (_enabled) { @@ -96,7 +110,7 @@ void Faceshift::noteError(QAbstractSocket::SocketError error) { } void Faceshift::readFromSocket() { - QByteArray buffer = _socket.readAll(); + QByteArray buffer = static_cast(sender())->readAll(); _stream.received(buffer.size(), buffer.constData()); fsMsgPtr msg; for (fsMsgPtr msg; (msg = _stream.get_message()); ) { @@ -151,8 +165,9 @@ void Faceshift::readFromSocket() { break; } } + _lastMessageReceived = usecTimestampNow(); } void Faceshift::send(const std::string& message) { - _socket.write(message.data(), message.size()); + _tcpSocket.write(message.data(), message.size()); } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 586f336c04..f93bbba2b4 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -10,6 +10,7 @@ #define __interface__Faceshift__ #include +#include #include #include @@ -24,7 +25,7 @@ public: Faceshift(); - bool isActive() const { return _socket.state() == QAbstractSocket::ConnectedState && _tracking; } + bool isActive() const; const glm::quat& getHeadRotation() const { return _headRotation; } const glm::vec3& getHeadTranslation() const { return _headTranslation; } @@ -63,10 +64,12 @@ private: void send(const std::string& message); - QTcpSocket _socket; + QTcpSocket _tcpSocket; + QUdpSocket _udpSocket; fs::fsBinaryStream _stream; bool _enabled; bool _tracking; + uint64_t _lastMessageReceived; glm::quat _headRotation; glm::vec3 _headTranslation; From b9878bbbd68228e44d8cf1e2b7d0274c0e7cd11d Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 16:55:59 -0700 Subject: [PATCH 2/5] Adjustment to UDP code. --- interface/src/devices/Faceshift.cpp | 32 ++++++++++++++++++++--------- interface/src/devices/Faceshift.h | 2 ++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 66ba958bbd..0df7ac79c4 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -26,12 +26,12 @@ Faceshift::Faceshift() : _eyeGazeRightYaw(0.0f), _leftBlink(0.0f), _rightBlink(0.0f), - _leftBlinkIndex(-1), - _rightBlinkIndex(-1), + _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes + _rightBlinkIndex(1), _browHeight(0.0f), - _browUpCenterIndex(-1), + _browUpCenterIndex(16), _mouthSize(0.0f), - _jawOpenIndex(-1), + _jawOpenIndex(21), _longTermAverageEyePitch(0.0f), _longTermAverageEyeYaw(0.0f), _estimatedEyePitch(0.0f), @@ -41,7 +41,7 @@ Faceshift::Faceshift() : connect(&_tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); connect(&_tcpSocket, SIGNAL(readyRead()), SLOT(readFromSocket())); - connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readFromSocket())); + connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); _udpSocket.bind(FACESHIFT_PORT); } @@ -109,8 +109,24 @@ void Faceshift::noteError(QAbstractSocket::SocketError error) { } } +void Faceshift::readPendingDatagrams() { + QByteArray buffer; + while (_udpSocket.hasPendingDatagrams()) { + buffer.resize(_udpSocket.pendingDatagramSize()); + _udpSocket.readDatagram(buffer.data(), buffer.size()); + receive(buffer); + } +} + void Faceshift::readFromSocket() { - QByteArray buffer = static_cast(sender())->readAll(); + receive(_tcpSocket.readAll()); +} + +void Faceshift::send(const std::string& message) { + _tcpSocket.write(message.data(), message.size()); +} + +void Faceshift::receive(const QByteArray& buffer) { _stream.received(buffer.size(), buffer.constData()); fsMsgPtr msg; for (fsMsgPtr msg; (msg = _stream.get_message()); ) { @@ -167,7 +183,3 @@ void Faceshift::readFromSocket() { } _lastMessageReceived = usecTimestampNow(); } - -void Faceshift::send(const std::string& message) { - _tcpSocket.write(message.data(), message.size()); -} diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index f93bbba2b4..f8ec6f122d 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -58,11 +58,13 @@ private slots: void connectSocket(); void noteConnected(); void noteError(QAbstractSocket::SocketError error); + void readPendingDatagrams(); void readFromSocket(); private: void send(const std::string& message); + void receive(const QByteArray& buffer); QTcpSocket _tcpSocket; QUdpSocket _udpSocket; From 4b19e3d0fe348b2a09e5863ba186e6f8a8c8c474 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 10 Sep 2013 17:43:06 -0700 Subject: [PATCH 3/5] Incorporate the "eye open" coefficients for eye widening. --- interface/src/devices/Faceshift.cpp | 15 ++++++++++++--- interface/src/devices/Faceshift.h | 3 +++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 0df7ac79c4..5da06ae912 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -28,6 +28,8 @@ Faceshift::Faceshift() : _rightBlink(0.0f), _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes _rightBlinkIndex(1), + _leftEyeOpenIndex(8), + _rightEyeOpenIndex(9), _browHeight(0.0f), _browUpCenterIndex(16), _mouthSize(0.0f), @@ -144,11 +146,12 @@ void Faceshift::receive(const QByteArray& buffer) { _eyeGazeRightPitch = -data.m_eyeGazeRightPitch; _eyeGazeRightYaw = data.m_eyeGazeRightYaw; + const float EYE_OPEN_SCALE = 0.25f; if (_leftBlinkIndex != -1) { - _leftBlink = data.m_coeffs[_leftBlinkIndex]; + _leftBlink = data.m_coeffs[_leftBlinkIndex] - data.m_coeffs[_leftEyeOpenIndex] * EYE_OPEN_SCALE; } if (_rightBlinkIndex != -1) { - _rightBlink = data.m_coeffs[_rightBlinkIndex]; + _rightBlink = data.m_coeffs[_rightBlinkIndex] - data.m_coeffs[_rightEyeOpenIndex] * EYE_OPEN_SCALE; } if (_browUpCenterIndex != -1) { _browHeight = data.m_coeffs[_browUpCenterIndex]; @@ -167,7 +170,13 @@ void Faceshift::receive(const QByteArray& buffer) { } else if (names[i] == "EyeBlink_R") { _rightBlinkIndex = i; - + + } else if (names[i] == "EyeOpen_L") { + _leftEyeOpenIndex = i; + + } else if (names[i] == "EyeOpen_R") { + _rightEyeOpenIndex = i; + } else if (names[i] == "BrowsU_C") { _browUpCenterIndex = i; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index f8ec6f122d..1bdb0cde59 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -88,6 +88,9 @@ private: int _leftBlinkIndex; int _rightBlinkIndex; + int _leftEyeOpenIndex; + int _rightEyeOpenIndex; + float _browHeight; int _browUpCenterIndex; From feecb4fa25daf267ae81068aed23951c04f7ada2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 11 Sep 2013 10:44:38 -0700 Subject: [PATCH 4/5] Fix for eyelids, change Faceshift menu option to indicate that it applies specifically to TCP connection. --- interface/src/Menu.cpp | 4 ++-- interface/src/Menu.h | 2 +- interface/src/avatar/Head.cpp | 4 ++-- interface/src/devices/Faceshift.cpp | 10 +++++----- interface/src/devices/Faceshift.h | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 1d9e12cc96..7e9dfe2355 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -370,11 +370,11 @@ Menu::Menu() : SLOT(setDepthOnly(bool))); addCheckableActionToQMenuAndActionHash(developerMenu, - MenuOption::Faceshift, + MenuOption::FaceshiftTCP, 0, false, appInstance->getFaceshift(), - SLOT(setEnabled(bool))); + SLOT(setTCPEnabled(bool))); QMenu* audioDebugMenu = developerMenu->addMenu("Audio Debugging Tools"); addCheckableActionToQMenuAndActionHash(audioDebugMenu, MenuOption::EchoAudio); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 934cb1b7ae..2f486da031 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -137,7 +137,7 @@ namespace MenuOption { const QString ExportVoxels = "Export Voxels"; const QString HeadMouse = "Head Mouse"; const QString FaceMode = "Cycle Face Mode"; - const QString Faceshift = "Faceshift"; + const QString FaceshiftTCP = "Faceshift (TCP)"; const QString FalseColorByDistance = "FALSE Color By Distance"; const QString FalseColorBySource = "FALSE Color By Source"; const QString FalseColorEveryOtherVoxel = "FALSE Color Every Other Randomly"; diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 517d062494..b439d56568 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -726,7 +726,7 @@ void Head::renderEyeBalls() { float angle = -67.5f - 50.0f * _leftEyeBlink; glRotatef(angle, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); - glRotatef(glm::mix(-angle, 180.0f, _leftEyeBlink), 1, 0, 0); + glRotatef(glm::mix(-angle, 180.0f, max(0.0f, _leftEyeBlink)), 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); } glPopMatrix(); @@ -740,7 +740,7 @@ void Head::renderEyeBalls() { float angle = -67.5f - 50.0f * _rightEyeBlink; glRotatef(angle, 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); - glRotatef(glm::mix(-angle, 180.0f, _rightEyeBlink), 1, 0, 0); + glRotatef(glm::mix(-angle, 180.0f, max(0.0f, _rightEyeBlink)), 1, 0, 0); Application::getInstance()->getGeometryCache()->renderHemisphere(15, 10); } glPopMatrix(); diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 5da06ae912..4bcab43cf3 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -18,7 +18,7 @@ using namespace std; const quint16 FACESHIFT_PORT = 33433; Faceshift::Faceshift() : - _enabled(false), + _tcpEnabled(false), _lastMessageReceived(0), _eyeGazeLeftPitch(0.0f), _eyeGazeLeftYaw(0.0f), @@ -75,8 +75,8 @@ void Faceshift::reset() { } } -void Faceshift::setEnabled(bool enabled) { - if ((_enabled = enabled)) { +void Faceshift::setTCPEnabled(bool enabled) { + if ((_tcpEnabled = enabled)) { connectSocket(); } else { @@ -85,7 +85,7 @@ void Faceshift::setEnabled(bool enabled) { } void Faceshift::connectSocket() { - if (_enabled) { + if (_tcpEnabled) { qDebug("Faceshift: Connecting...\n"); _tcpSocket.connectToHost("localhost", FACESHIFT_PORT); @@ -106,7 +106,7 @@ void Faceshift::noteError(QAbstractSocket::SocketError error) { qDebug() << "Faceshift: " << _tcpSocket.errorString() << "\n"; // reconnect after a delay - if (_enabled) { + if (_tcpEnabled) { QTimer::singleShot(1000, this, SLOT(connectSocket())); } } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 1bdb0cde59..29a25f9c4c 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -51,7 +51,7 @@ public: public slots: - void setEnabled(bool enabled); + void setTCPEnabled(bool enabled); private slots: @@ -69,7 +69,7 @@ private: QTcpSocket _tcpSocket; QUdpSocket _udpSocket; fs::fsBinaryStream _stream; - bool _enabled; + bool _tcpEnabled; bool _tracking; uint64_t _lastMessageReceived; From 80cc278bfaf7a1acee7d81a6b00ae4b24504cb26 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 11 Sep 2013 10:49:12 -0700 Subject: [PATCH 5/5] Increase the eye open scale. --- interface/src/devices/Faceshift.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 4bcab43cf3..bb4538b0e8 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -146,7 +146,7 @@ void Faceshift::receive(const QByteArray& buffer) { _eyeGazeRightPitch = -data.m_eyeGazeRightPitch; _eyeGazeRightYaw = data.m_eyeGazeRightYaw; - const float EYE_OPEN_SCALE = 0.25f; + const float EYE_OPEN_SCALE = 0.5f; if (_leftBlinkIndex != -1) { _leftBlink = data.m_coeffs[_leftBlinkIndex] - data.m_coeffs[_leftEyeOpenIndex] * EYE_OPEN_SCALE; }