From f9426cbecc461a7b6c275d95974b16eb98286c8f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 16 Sep 2013 15:49:40 -0700 Subject: [PATCH] Working on reading and rendering the rig from Faceshift. --- interface/src/Menu.cpp | 3 +- interface/src/Menu.h | 3 + interface/src/avatar/BlendFace.cpp | 94 +++++++++++++++++++++++++++++ interface/src/avatar/BlendFace.h | 47 +++++++++++++++ interface/src/avatar/Head.cpp | 3 +- interface/src/avatar/Head.h | 5 +- interface/src/avatar/MyAvatar.cpp | 10 ++- interface/src/avatar/MyAvatar.h | 3 +- interface/src/devices/Faceshift.cpp | 27 ++++++++- interface/src/devices/Faceshift.h | 7 ++- 10 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 interface/src/avatar/BlendFace.cpp create mode 100644 interface/src/avatar/BlendFace.h diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 8a693582c1..44bb2e24ff 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -275,7 +275,8 @@ Menu::Menu() : appInstance->getGlowEffect(), SLOT(cycleRenderMode())); - + addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::UseFaceshiftRig, 0, false, + appInstance->getFaceshift(), SLOT(setUsingRig(bool))); addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::UsePerlinFace, 0, false); addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::LookAtVectors, 0, true); addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::LookAtIndicator, 0, true); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 76e2f945ce..081810d374 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -31,6 +31,8 @@ struct ViewFrustumOffset { float up; }; +class QSettings; + class BandwidthDialog; class VoxelStatsDialog; @@ -197,6 +199,7 @@ namespace MenuOption { const QString TestRaveGlove = "Test Rave Glove"; const QString TreeStats = "Calculate Tree Stats"; const QString TransmitterDrive = "Transmitter Drive"; + const QString UseFaceshiftRig = "Use Faceshift Rig"; const QString UsePerlinFace = "Use Perlin's Face"; const QString Quit = "Quit"; const QString Webcam = "Webcam"; diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp new file mode 100644 index 0000000000..18c948f8cc --- /dev/null +++ b/interface/src/avatar/BlendFace.cpp @@ -0,0 +1,94 @@ +// +// BlendFace.cpp +// interface +// +// Created by Andrzej Kapolka on 9/16/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include "BlendFace.h" +#include "Head.h" + +using namespace fs; + +BlendFace::BlendFace(Head* owningHead) : + _owningHead(owningHead), + _iboID(0) +{ +} + +BlendFace::~BlendFace() { + if (_iboID != 0) { + glDeleteBuffers(1, &_iboID); + glDeleteBuffers(1, &_vboID); + } +} + +bool BlendFace::render(float alpha) { + if (_iboID == 0) { + return false; + } + + glPushMatrix(); + glTranslatef(_owningHead->getPosition().x, _owningHead->getPosition().y, _owningHead->getPosition().z); + glm::quat orientation = _owningHead->getOrientation(); + glm::vec3 axis = glm::axis(orientation); + glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); + glScalef(_owningHead->getScale(), _owningHead->getScale(), _owningHead->getScale()); + + glColor4f(1.0f, 1.0f, 1.0f, alpha); + + // update the blended vertices + glBindBuffer(GL_ARRAY_BUFFER, _vboID); + glBufferSubData(GL_ARRAY_BUFFER, 0, _baseVertices.size() * sizeof(fsVector3f), _baseVertices.data()); + + // tell OpenGL where to find vertex information + glEnableClientState(GL_VERTEX_ARRAY); + glVertexPointer(3, GL_FLOAT, 0, 0); + + glPolygonMode(GL_FRONT_AND_BACK, GL_LINE); + + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); + glDrawRangeElementsEXT(GL_TRIANGLES, 0, _baseVertices.size() - 1, _indexCount, GL_UNSIGNED_INT, 0); + + glPolygonMode(GL_FRONT_AND_BACK, GL_FILL); + + // deactivate vertex arrays after drawing + glDisableClientState(GL_VERTEX_ARRAY); + + // bind with 0 to switch back to normal operation + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + + glPopMatrix(); + + return true; +} + +void BlendFace::setRig(const fsMsgRig& rig) { + if (rig.mesh().m_tris.empty()) { + // clear any existing geometry + if (_iboID != 0) { + glDeleteBuffers(1, &_iboID); + glDeleteBuffers(1, &_vboID); + _iboID = 0; + } + return; + } + if (_iboID == 0) { + glGenBuffers(1, &_iboID); + glGenBuffers(1, &_vboID); + } + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); + glBufferData(GL_ELEMENT_ARRAY_BUFFER, rig.mesh().m_tris.size() * sizeof(fsVector3i), + rig.mesh().m_tris.data(), GL_STATIC_DRAW); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + + glBindBuffer(GL_ARRAY_BUFFER, _vboID); + glBufferData(GL_ARRAY_BUFFER, rig.mesh().m_vertex_data.m_vertices.size() * sizeof(fsVector3f), NULL, GL_DYNAMIC_DRAW); + glBindBuffer(GL_ARRAY_BUFFER, 0); + + _indexCount = rig.mesh().m_tris.size() * 3; + _baseVertices = rig.mesh().m_vertex_data.m_vertices; + _blendshapes = rig.blendshapes(); +} diff --git a/interface/src/avatar/BlendFace.h b/interface/src/avatar/BlendFace.h new file mode 100644 index 0000000000..9385cf4e0b --- /dev/null +++ b/interface/src/avatar/BlendFace.h @@ -0,0 +1,47 @@ +// +// BlendFace.h +// interface +// +// Created by Andrzej Kapolka on 9/16/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__BlendFace__ +#define __interface__BlendFace__ + +#include + +#include + +#include "InterfaceConfig.h" + +class Head; + +/// A face formed from a linear mix of blendshapes according to a set of coefficients. +class BlendFace : public QObject { + Q_OBJECT + +public: + + BlendFace(Head* owningHead); + ~BlendFace(); + + bool render(float alpha); + +private slots: + + void setRig(const fs::fsMsgRig& rig); + +private: + + Head* _owningHead; + + GLuint _iboID; + GLuint _vboID; + + GLsizei _indexCount; + std::vector _baseVertices; + std::vector _blendshapes; +}; + +#endif /* defined(__interface__BlendFace__) */ diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index af6133460f..9ebd238990 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -87,7 +87,8 @@ Head::Head(Avatar* owningAvatar) : _cameraFollowsHead(false), _cameraFollowHeadRate(0.0f), _face(this), - _perlinFace(this) + _perlinFace(this), + _blendFace(this) { if (USING_PHYSICAL_MOHAWK) { resetHairPhysics(); diff --git a/interface/src/avatar/Head.h b/interface/src/avatar/Head.h index 496fdecef6..87cb47bb2d 100644 --- a/interface/src/avatar/Head.h +++ b/interface/src/avatar/Head.h @@ -18,9 +18,10 @@ #include #include "BendyLine.h" +#include "BlendFace.h" #include "Face.h" -#include "PerlinFace.h" #include "InterfaceConfig.h" +#include "PerlinFace.h" #include "world.h" #include "devices/SerialInterface.h" @@ -71,6 +72,7 @@ public: glm::vec3 getFrontDirection() const { return getOrientation() * IDENTITY_FRONT; } Face& getFace() { return _face; } + BlendFace& getBlendFace() { return _blendFace; } const bool getReturnToCenter() const { return _returnHeadToCenter; } // Do you want head to try to return to center (depends on interface detected) float getAverageLoudness() const { return _averageLoudness; } @@ -128,6 +130,7 @@ private: float _cameraFollowHeadRate; Face _face; PerlinFace _perlinFace; + BlendFace _blendFace; static ProgramObject _irisProgram; static GLuint _irisTextureID; diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 0dcfe32236..e05e236059 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -56,7 +56,15 @@ MyAvatar::MyAvatar(Node* owningNode) : _driveKeys[i] = false; } - _collisionRadius = _height * COLLISION_RADIUS_SCALE; + _collisionRadius = _height * COLLISION_RADIUS_SCALE; +} + +void MyAvatar::init() { + Avatar::init(); + + // when we receive a Faceshift rig, apply it to our own blend face + _head.getBlendFace().connect(Application::getInstance()->getFaceshift(), SIGNAL(rigReceived(fs::fsMsgRig)), + SLOT(setRig(fs::fsMsgRig))); } void MyAvatar::reset() { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index b88b5befcf..cadc55018c 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -15,6 +15,7 @@ class MyAvatar : public Avatar { public: MyAvatar(Node* owningNode = NULL); + void init(); void reset(); void simulate(float deltaTime, Transmitter* transmitter, float gyroCameraSensitivity); void updateFromGyrosAndOrWebcam(bool gyroLook, float pitchFromTouch); @@ -87,4 +88,4 @@ public: void checkForMouseRayTouching(); }; -#endif \ No newline at end of file +#endif diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index cc86785e60..0e21f91863 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -11,6 +11,7 @@ #include #include "Faceshift.h" +#include "Menu.h" using namespace fs; using namespace std; @@ -98,6 +99,17 @@ void Faceshift::setTCPEnabled(bool enabled) { } } +void Faceshift::setUsingRig(bool usingRig) { + if (usingRig && _tcpSocket.state() == QAbstractSocket::ConnectedState) { + string message; + fsBinaryStream::encode_message(message, fsMsgSendRig()); + send(message); + + } else { + emit rigReceived(fsMsgRig()); + } +} + void Faceshift::connectSocket() { if (_tcpEnabled) { qDebug("Faceshift: Connecting...\n"); @@ -114,6 +126,11 @@ void Faceshift::noteConnected() { string message; fsBinaryStream::encode_message(message, fsMsgSendBlendshapeNames()); send(message); + + // if using faceshift rig, request it + if (Menu::getInstance()->isOptionChecked(MenuOption::UseFaceshiftRig)) { + setUsingRig(true); + } } void Faceshift::noteError(QAbstractSocket::SocketError error) { @@ -208,10 +225,10 @@ void Faceshift::receive(const QByteArray& buffer) { } else if (names[i] == "EyeBlink_R") { _rightBlinkIndex = i; - }else if (names[i] == "EyeOpen_L") { + } else if (names[i] == "EyeOpen_L") { _leftEyeOpenIndex = i; - }else if (names[i] == "EyeOpen_R") { + } else if (names[i] == "EyeOpen_R") { _rightEyeOpenIndex = i; } else if (names[i] == "BrowsD_L") { @@ -237,11 +254,15 @@ void Faceshift::receive(const QByteArray& buffer) { } else if (names[i] == "MouthSmile_R") { _mouthSmileRightIndex = i; - } } break; } + case fsMsg::MSG_OUT_RIG: { + fsMsgRig* rig = static_cast(msg.get()); + emit rigReceived(*rig); + break; + } default: break; } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 6e403039af..c6ab54e694 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -57,10 +57,15 @@ public: void update(); void reset(); +signals: + + void rigReceived(const fs::fsMsgRig& rig); + public slots: void setTCPEnabled(bool enabled); - + void setUsingRig(bool usingRig); + private slots: void connectSocket();