From 1fce6c717b70d85df08cd726f7693dfe8d4e4ed3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 16 Sep 2013 17:28:04 -0700 Subject: [PATCH] Working on shipping around the complete set of blendshape coefficients. --- interface/src/avatar/Head.cpp | 3 +- interface/src/devices/Faceshift.cpp | 68 +++++--------------------- interface/src/devices/Faceshift.h | 49 ++++++++----------- libraries/avatars/src/AvatarData.cpp | 10 ++++ libraries/avatars/src/HeadData.h | 4 ++ libraries/shared/src/PacketHeaders.cpp | 2 +- 6 files changed, 49 insertions(+), 87 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 9ebd238990..e489bf89ed 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -160,6 +160,7 @@ void Head::simulate(float deltaTime, bool isMine, float gyroCameraSensitivity) { _averageLoudness = faceshift->getMouthSize() * faceshift->getMouthSize() * MOUTH_SIZE_SCALE; const float BROW_HEIGHT_SCALE = 0.005f; _browAudioLift = faceshift->getBrowUpCenter() * BROW_HEIGHT_SCALE; + _blendshapeCoefficients = faceshift->getBlendshapeCoefficients(); } else if (!_isFaceshiftConnected) { // Update eye saccades @@ -326,7 +327,7 @@ void Head::calculateGeometry() { void Head::render(float alpha) { _renderAlpha = alpha; - if (!_face.render(alpha)) { + if (!(_face.render(alpha) || _blendFace.render(alpha))) { calculateGeometry(); glEnable(GL_DEPTH_TEST); diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 0e21f91863..f62bf239bc 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -25,25 +25,13 @@ Faceshift::Faceshift() : _eyeGazeLeftYaw(0.0f), _eyeGazeRightPitch(0.0f), _eyeGazeRightYaw(0.0f), - _leftBlink(0.0f), - _rightBlink(0.0f), - _leftEyeOpen(0.0f), - _rightEyeOpen(0.0f), - _browDownLeft(0.0f), - _browDownRight(0.0f), - _browUpCenter(0.0f), - _browUpLeft(0.0f), - _browUpRight(0.0f), - _browDownLeftIndex(-1), - _browDownRightIndex(-1), - _browUpLeftIndex(-1), - _browUpRightIndex(-1), - _mouthSize(0.0f), - _mouthSmileLeft(0), - _mouthSmileRight(0), - _mouthSmileLeftIndex(-1), - _mouthSmileRightIndex(0), - _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes + _browDownLeftIndex(14), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes + _browDownRightIndex(15), + _browUpLeftIndex(17), + _browUpRightIndex(18), + _mouthSmileLeftIndex(28), + _mouthSmileRightIndex(29), + _leftBlinkIndex(0), _rightBlinkIndex(1), _leftEyeOpenIndex(8), _rightEyeOpenIndex(9), @@ -155,6 +143,10 @@ void Faceshift::readFromSocket() { receive(_tcpSocket.readAll()); } +float Faceshift::getBlendshapeCoefficient(int index) const { + return (index >= 0 && index < _blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; +} + void Faceshift::send(const std::string& message) { _tcpSocket.write(message.data(), message.size()); } @@ -176,43 +168,7 @@ void Faceshift::receive(const QByteArray& buffer) { _eyeGazeLeftYaw = data.m_eyeGazeLeftYaw; _eyeGazeRightPitch = -data.m_eyeGazeRightPitch; _eyeGazeRightYaw = data.m_eyeGazeRightYaw; - - if (_leftBlinkIndex != -1) { - _leftBlink = data.m_coeffs[_leftBlinkIndex]; - } - if (_rightBlinkIndex != -1) { - _rightBlink = data.m_coeffs[_rightBlinkIndex]; - } - if (_leftEyeOpenIndex != -1) { - _leftEyeOpen = data.m_coeffs[_leftEyeOpenIndex]; - } - if (_rightEyeOpenIndex != -1) { - _rightEyeOpen = data.m_coeffs[_rightEyeOpenIndex]; - } - if (_browDownLeftIndex != -1) { - _browDownLeft = data.m_coeffs[_browDownLeftIndex]; - } - if (_browDownRightIndex != -1) { - _browDownRight = data.m_coeffs[_browDownRightIndex]; - } - if (_browUpCenterIndex != -1) { - _browUpCenter = data.m_coeffs[_browUpCenterIndex]; - } - if (_browUpLeftIndex != -1) { - _browUpLeft = data.m_coeffs[_browUpLeftIndex]; - } - if (_browUpRightIndex != -1) { - _browUpRight = data.m_coeffs[_browUpRightIndex]; - } - if (_jawOpenIndex != -1) { - _mouthSize = data.m_coeffs[_jawOpenIndex]; - } - if (_mouthSmileLeftIndex != -1) { - _mouthSmileLeft = data.m_coeffs[_mouthSmileLeftIndex]; - } - if (_mouthSmileRightIndex != -1) { - _mouthSmileRight = data.m_coeffs[_mouthSmileRightIndex]; - } + _blendshapeCoefficients = data.m_coeffs; } break; } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index c6ab54e694..dc42a4e9c6 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -9,6 +9,8 @@ #ifndef __interface__Faceshift__ #define __interface__Faceshift__ +#include + #include #include @@ -39,20 +41,22 @@ public: float getEstimatedEyePitch() const { return _estimatedEyePitch; } float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } - float getLeftBlink() const { return _leftBlink; } - float getRightBlink() const { return _rightBlink; } - float getLeftEyeOpen() const { return _leftEyeOpen; } - float getRightEyeOpen() const { return _rightEyeOpen; } + const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } - float getBrowDownLeft() const { return _browDownLeft; } - float getBrowDownRight() const { return _browDownRight; } - float getBrowUpCenter() const { return _browUpCenter; } - float getBrowUpLeft() const { return _browUpLeft; } - float getBrowUpRight() const { return _browUpRight; } + float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } + float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } + float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } + float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); } - float getMouthSize() const { return _mouthSize; } - float getMouthSmileLeft() const { return _mouthSmileLeft; } - float getMouthSmileRight() const { return _mouthSmileRight; } + float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); } + float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); } + float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); } + float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); } + float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); } + + float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); } + float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } + float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } void update(); void reset(); @@ -76,6 +80,8 @@ private slots: private: + float getBlendshapeCoefficient(int index) const; + void send(const std::string& message); void receive(const QByteArray& buffer); @@ -95,35 +101,20 @@ private: float _eyeGazeRightPitch; float _eyeGazeRightYaw; - float _leftBlink; - float _rightBlink; - float _leftEyeOpen; - float _rightEyeOpen; - + std::vector _blendshapeCoefficients; + int _leftBlinkIndex; int _rightBlinkIndex; int _leftEyeOpenIndex; int _rightEyeOpenIndex; // Brows - float _browDownLeft; - float _browDownRight; - float _browUpCenter; - float _browUpLeft; - float _browUpRight; - int _browDownLeftIndex; int _browDownRightIndex; - int _browUpCenterIndex; int _browUpLeftIndex; int _browUpRightIndex; - float _mouthSize; - - float _mouthSmileLeft; - float _mouthSmileRight; - int _mouthSmileLeftIndex; int _mouthSmileRightIndex; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index cb445ffaa7..dab3d1651e 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -204,6 +204,11 @@ int AvatarData::getBroadcastData(unsigned char* destinationBuffer) { memcpy(destinationBuffer, &_headData->_browAudioLift, sizeof(float)); destinationBuffer += sizeof(float); + + *destinationBuffer++ = _headData->_blendshapeCoefficients.size(); + memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), + _headData->_blendshapeCoefficients.size() * sizeof(float)); + destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); } // leap hand data @@ -334,6 +339,11 @@ int AvatarData::parseData(unsigned char* sourceBuffer, int numBytes) { memcpy(&_headData->_browAudioLift, sourceBuffer, sizeof(float)); sourceBuffer += sizeof(float); + + _headData->_blendshapeCoefficients.resize(*sourceBuffer++); + memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, + _headData->_blendshapeCoefficients.size() * sizeof(float)); + sourceBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); } // leap hand data diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index dd4d5b61a4..7855c1eb68 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -10,6 +10,7 @@ #define __hifi__HeadData__ #include +#include #include @@ -52,6 +53,7 @@ public: void setLookAtPosition(const glm::vec3& lookAtPosition) { _lookAtPosition = lookAtPosition; } friend class AvatarData; + protected: float _yaw; float _pitch; @@ -65,7 +67,9 @@ protected: float _rightEyeBlink; float _averageLoudness; float _browAudioLift; + std::vector _blendshapeCoefficients; AvatarData* _owningAvatar; + private: // privatize copy ctor and assignment operator so copies of this object cannot be made HeadData(const HeadData&); diff --git a/libraries/shared/src/PacketHeaders.cpp b/libraries/shared/src/PacketHeaders.cpp index fd261e4385..ff33f35aca 100644 --- a/libraries/shared/src/PacketHeaders.cpp +++ b/libraries/shared/src/PacketHeaders.cpp @@ -20,7 +20,7 @@ PACKET_VERSION versionForPacketType(PACKET_TYPE type) { return 1; case PACKET_TYPE_HEAD_DATA: - return 6; + return 7; case PACKET_TYPE_AVATAR_FACE_VIDEO: return 1;