From f9426cbecc461a7b6c275d95974b16eb98286c8f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 16 Sep 2013 15:49:40 -0700 Subject: [PATCH 01/16] Working on reading and rendering the rig from Faceshift. --- interface/src/Menu.cpp | 3 +- interface/src/Menu.h | 3 + interface/src/avatar/BlendFace.cpp | 94 +++++++++++++++++++++++++++++ interface/src/avatar/BlendFace.h | 47 +++++++++++++++ interface/src/avatar/Head.cpp | 3 +- interface/src/avatar/Head.h | 5 +- interface/src/avatar/MyAvatar.cpp | 10 ++- interface/src/avatar/MyAvatar.h | 3 +- interface/src/devices/Faceshift.cpp | 27 ++++++++- interface/src/devices/Faceshift.h | 7 ++- 10 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 interface/src/avatar/BlendFace.cpp create mode 100644 interface/src/avatar/BlendFace.h diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 8a693582c1..44bb2e24ff 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -275,7 +275,8 @@ Menu::Menu() : appInstance->getGlowEffect(), SLOT(cycleRenderMode())); - + addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::UseFaceshiftRig, 0, false, + appInstance->getFaceshift(), SLOT(setUsingRig(bool))); addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::UsePerlinFace, 0, false); addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::LookAtVectors, 0, true); addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::LookAtIndicator, 0, true); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 76e2f945ce..081810d374 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -31,6 +31,8 @@ struct ViewFrustumOffset { float up; }; +class QSettings; + class BandwidthDialog; class VoxelStatsDialog; @@ -197,6 +199,7 @@ namespace MenuOption { const QString TestRaveGlove = "Test Rave Glove"; const QString TreeStats = "Calculate Tree Stats"; const QString TransmitterDrive = "Transmitter Drive"; + const QString UseFaceshiftRig = "Use Faceshift Rig"; const QString UsePerlinFace = "Use Perlin's Face"; const QString Quit = "Quit"; const QString Webcam = "Webcam"; diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp new file mode 100644 index 0000000000..18c948f8cc --- /dev/null +++ b/interface/src/avatar/BlendFace.cpp @@ -0,0 +1,94 @@ +// +// BlendFace.cpp +// interface +// +// Created by Andrzej Kapolka on 9/16/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include "BlendFace.h" +#include "Head.h" + +using namespace fs; + +BlendFace::BlendFace(Head* owningHead) : + _owningHead(owningHead), + _iboID(0) +{ +} + +BlendFace::~BlendFace() { + if (_iboID != 0) { + glDeleteBuffers(1, &_iboID); + glDeleteBuffers(1, &_vboID); + } +} + +bool BlendFace::render(float alpha) { + if (_iboID == 0) { + return false; + } + + glPushMatrix(); + glTranslatef(_owningHead->getPosition().x, _owningHead->getPosition().y, _owningHead->getPosition().z); + glm::quat orientation = _owningHead->getOrientation(); + glm::vec3 axis = glm::axis(orientation); + glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); + glScalef(_owningHead->getScale(), _owningHead->getScale(), _owningHead->getScale()); + + glColor4f(1.0f, 1.0f, 1.0f, alpha); + + // update the blended vertices + glBindBuffer(GL_ARRAY_BUFFER, _vboID); + glBufferSubData(GL_ARRAY_BUFFER, 0, _baseVertices.size() * sizeof(fsVector3f), _baseVertices.data()); + + // tell OpenGL where to find vertex information + glEnableClientState(GL_VERTEX_ARRAY); + glVertexPointer(3, GL_FLOAT, 0, 0); + + glPolygonMode(GL_FRONT_AND_BACK, GL_LINE); + + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); + glDrawRangeElementsEXT(GL_TRIANGLES, 0, _baseVertices.size() - 1, _indexCount, GL_UNSIGNED_INT, 0); + + glPolygonMode(GL_FRONT_AND_BACK, GL_FILL); + + // deactivate vertex arrays after drawing + glDisableClientState(GL_VERTEX_ARRAY); + + // bind with 0 to switch back to normal operation + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + + glPopMatrix(); + + return true; +} + +void BlendFace::setRig(const fsMsgRig& rig) { + if (rig.mesh().m_tris.empty()) { + // clear any existing geometry + if (_iboID != 0) { + glDeleteBuffers(1, &_iboID); + glDeleteBuffers(1, &_vboID); + _iboID = 0; + } + return; + } + if (_iboID == 0) { + glGenBuffers(1, &_iboID); + glGenBuffers(1, &_vboID); + } + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); + glBufferData(GL_ELEMENT_ARRAY_BUFFER, rig.mesh().m_tris.size() * sizeof(fsVector3i), + rig.mesh().m_tris.data(), GL_STATIC_DRAW); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + + glBindBuffer(GL_ARRAY_BUFFER, _vboID); + glBufferData(GL_ARRAY_BUFFER, rig.mesh().m_vertex_data.m_vertices.size() * sizeof(fsVector3f), NULL, GL_DYNAMIC_DRAW); + glBindBuffer(GL_ARRAY_BUFFER, 0); + + _indexCount = rig.mesh().m_tris.size() * 3; + _baseVertices = rig.mesh().m_vertex_data.m_vertices; + _blendshapes = rig.blendshapes(); +} diff --git a/interface/src/avatar/BlendFace.h b/interface/src/avatar/BlendFace.h new file mode 100644 index 0000000000..9385cf4e0b --- /dev/null +++ b/interface/src/avatar/BlendFace.h @@ -0,0 +1,47 @@ +// +// BlendFace.h +// interface +// +// Created by Andrzej Kapolka on 9/16/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__BlendFace__ +#define __interface__BlendFace__ + +#include + +#include + +#include "InterfaceConfig.h" + +class Head; + +/// A face formed from a linear mix of blendshapes according to a set of coefficients. +class BlendFace : public QObject { + Q_OBJECT + +public: + + BlendFace(Head* owningHead); + ~BlendFace(); + + bool render(float alpha); + +private slots: + + void setRig(const fs::fsMsgRig& rig); + +private: + + Head* _owningHead; + + GLuint _iboID; + GLuint _vboID; + + GLsizei _indexCount; + std::vector _baseVertices; + std::vector _blendshapes; +}; + +#endif /* defined(__interface__BlendFace__) */ diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index af6133460f..9ebd238990 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -87,7 +87,8 @@ Head::Head(Avatar* owningAvatar) : _cameraFollowsHead(false), _cameraFollowHeadRate(0.0f), _face(this), - _perlinFace(this) + _perlinFace(this), + _blendFace(this) { if (USING_PHYSICAL_MOHAWK) { resetHairPhysics(); diff --git a/interface/src/avatar/Head.h b/interface/src/avatar/Head.h index 496fdecef6..87cb47bb2d 100644 --- a/interface/src/avatar/Head.h +++ b/interface/src/avatar/Head.h @@ -18,9 +18,10 @@ #include #include "BendyLine.h" +#include "BlendFace.h" #include "Face.h" -#include "PerlinFace.h" #include "InterfaceConfig.h" +#include "PerlinFace.h" #include "world.h" #include "devices/SerialInterface.h" @@ -71,6 +72,7 @@ public: glm::vec3 getFrontDirection() const { return getOrientation() * IDENTITY_FRONT; } Face& getFace() { return _face; } + BlendFace& getBlendFace() { return _blendFace; } const bool getReturnToCenter() const { return _returnHeadToCenter; } // Do you want head to try to return to center (depends on interface detected) float getAverageLoudness() const { return _averageLoudness; } @@ -128,6 +130,7 @@ private: float _cameraFollowHeadRate; Face _face; PerlinFace _perlinFace; + BlendFace _blendFace; static ProgramObject _irisProgram; static GLuint _irisTextureID; diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 0dcfe32236..e05e236059 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -56,7 +56,15 @@ MyAvatar::MyAvatar(Node* owningNode) : _driveKeys[i] = false; } - _collisionRadius = _height * COLLISION_RADIUS_SCALE; + _collisionRadius = _height * COLLISION_RADIUS_SCALE; +} + +void MyAvatar::init() { + Avatar::init(); + + // when we receive a Faceshift rig, apply it to our own blend face + _head.getBlendFace().connect(Application::getInstance()->getFaceshift(), SIGNAL(rigReceived(fs::fsMsgRig)), + SLOT(setRig(fs::fsMsgRig))); } void MyAvatar::reset() { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index b88b5befcf..cadc55018c 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -15,6 +15,7 @@ class MyAvatar : public Avatar { public: MyAvatar(Node* owningNode = NULL); + void init(); void reset(); void simulate(float deltaTime, Transmitter* transmitter, float gyroCameraSensitivity); void updateFromGyrosAndOrWebcam(bool gyroLook, float pitchFromTouch); @@ -87,4 +88,4 @@ public: void checkForMouseRayTouching(); }; -#endif \ No newline at end of file +#endif diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index cc86785e60..0e21f91863 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -11,6 +11,7 @@ #include #include "Faceshift.h" +#include "Menu.h" using namespace fs; using namespace std; @@ -98,6 +99,17 @@ void Faceshift::setTCPEnabled(bool enabled) { } } +void Faceshift::setUsingRig(bool usingRig) { + if (usingRig && _tcpSocket.state() == QAbstractSocket::ConnectedState) { + string message; + fsBinaryStream::encode_message(message, fsMsgSendRig()); + send(message); + + } else { + emit rigReceived(fsMsgRig()); + } +} + void Faceshift::connectSocket() { if (_tcpEnabled) { qDebug("Faceshift: Connecting...\n"); @@ -114,6 +126,11 @@ void Faceshift::noteConnected() { string message; fsBinaryStream::encode_message(message, fsMsgSendBlendshapeNames()); send(message); + + // if using faceshift rig, request it + if (Menu::getInstance()->isOptionChecked(MenuOption::UseFaceshiftRig)) { + setUsingRig(true); + } } void Faceshift::noteError(QAbstractSocket::SocketError error) { @@ -208,10 +225,10 @@ void Faceshift::receive(const QByteArray& buffer) { } else if (names[i] == "EyeBlink_R") { _rightBlinkIndex = i; - }else if (names[i] == "EyeOpen_L") { + } else if (names[i] == "EyeOpen_L") { _leftEyeOpenIndex = i; - }else if (names[i] == "EyeOpen_R") { + } else if (names[i] == "EyeOpen_R") { _rightEyeOpenIndex = i; } else if (names[i] == "BrowsD_L") { @@ -237,11 +254,15 @@ void Faceshift::receive(const QByteArray& buffer) { } else if (names[i] == "MouthSmile_R") { _mouthSmileRightIndex = i; - } } break; } + case fsMsg::MSG_OUT_RIG: { + fsMsgRig* rig = static_cast(msg.get()); + emit rigReceived(*rig); + break; + } default: break; } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 6e403039af..c6ab54e694 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -57,10 +57,15 @@ public: void update(); void reset(); +signals: + + void rigReceived(const fs::fsMsgRig& rig); + public slots: void setTCPEnabled(bool enabled); - + void setUsingRig(bool usingRig); + private slots: void connectSocket(); From 1fce6c717b70d85df08cd726f7693dfe8d4e4ed3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 16 Sep 2013 17:28:04 -0700 Subject: [PATCH 02/16] Working on shipping around the complete set of blendshape coefficients. --- interface/src/avatar/Head.cpp | 3 +- interface/src/devices/Faceshift.cpp | 68 +++++--------------------- interface/src/devices/Faceshift.h | 49 ++++++++----------- libraries/avatars/src/AvatarData.cpp | 10 ++++ libraries/avatars/src/HeadData.h | 4 ++ libraries/shared/src/PacketHeaders.cpp | 2 +- 6 files changed, 49 insertions(+), 87 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 9ebd238990..e489bf89ed 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -160,6 +160,7 @@ void Head::simulate(float deltaTime, bool isMine, float gyroCameraSensitivity) { _averageLoudness = faceshift->getMouthSize() * faceshift->getMouthSize() * MOUTH_SIZE_SCALE; const float BROW_HEIGHT_SCALE = 0.005f; _browAudioLift = faceshift->getBrowUpCenter() * BROW_HEIGHT_SCALE; + _blendshapeCoefficients = faceshift->getBlendshapeCoefficients(); } else if (!_isFaceshiftConnected) { // Update eye saccades @@ -326,7 +327,7 @@ void Head::calculateGeometry() { void Head::render(float alpha) { _renderAlpha = alpha; - if (!_face.render(alpha)) { + if (!(_face.render(alpha) || _blendFace.render(alpha))) { calculateGeometry(); glEnable(GL_DEPTH_TEST); diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 0e21f91863..f62bf239bc 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -25,25 +25,13 @@ Faceshift::Faceshift() : _eyeGazeLeftYaw(0.0f), _eyeGazeRightPitch(0.0f), _eyeGazeRightYaw(0.0f), - _leftBlink(0.0f), - _rightBlink(0.0f), - _leftEyeOpen(0.0f), - _rightEyeOpen(0.0f), - _browDownLeft(0.0f), - _browDownRight(0.0f), - _browUpCenter(0.0f), - _browUpLeft(0.0f), - _browUpRight(0.0f), - _browDownLeftIndex(-1), - _browDownRightIndex(-1), - _browUpLeftIndex(-1), - _browUpRightIndex(-1), - _mouthSize(0.0f), - _mouthSmileLeft(0), - _mouthSmileRight(0), - _mouthSmileLeftIndex(-1), - _mouthSmileRightIndex(0), - _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes + _browDownLeftIndex(14), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes + _browDownRightIndex(15), + _browUpLeftIndex(17), + _browUpRightIndex(18), + _mouthSmileLeftIndex(28), + _mouthSmileRightIndex(29), + _leftBlinkIndex(0), _rightBlinkIndex(1), _leftEyeOpenIndex(8), _rightEyeOpenIndex(9), @@ -155,6 +143,10 @@ void Faceshift::readFromSocket() { receive(_tcpSocket.readAll()); } +float Faceshift::getBlendshapeCoefficient(int index) const { + return (index >= 0 && index < _blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f; +} + void Faceshift::send(const std::string& message) { _tcpSocket.write(message.data(), message.size()); } @@ -176,43 +168,7 @@ void Faceshift::receive(const QByteArray& buffer) { _eyeGazeLeftYaw = data.m_eyeGazeLeftYaw; _eyeGazeRightPitch = -data.m_eyeGazeRightPitch; _eyeGazeRightYaw = data.m_eyeGazeRightYaw; - - if (_leftBlinkIndex != -1) { - _leftBlink = data.m_coeffs[_leftBlinkIndex]; - } - if (_rightBlinkIndex != -1) { - _rightBlink = data.m_coeffs[_rightBlinkIndex]; - } - if (_leftEyeOpenIndex != -1) { - _leftEyeOpen = data.m_coeffs[_leftEyeOpenIndex]; - } - if (_rightEyeOpenIndex != -1) { - _rightEyeOpen = data.m_coeffs[_rightEyeOpenIndex]; - } - if (_browDownLeftIndex != -1) { - _browDownLeft = data.m_coeffs[_browDownLeftIndex]; - } - if (_browDownRightIndex != -1) { - _browDownRight = data.m_coeffs[_browDownRightIndex]; - } - if (_browUpCenterIndex != -1) { - _browUpCenter = data.m_coeffs[_browUpCenterIndex]; - } - if (_browUpLeftIndex != -1) { - _browUpLeft = data.m_coeffs[_browUpLeftIndex]; - } - if (_browUpRightIndex != -1) { - _browUpRight = data.m_coeffs[_browUpRightIndex]; - } - if (_jawOpenIndex != -1) { - _mouthSize = data.m_coeffs[_jawOpenIndex]; - } - if (_mouthSmileLeftIndex != -1) { - _mouthSmileLeft = data.m_coeffs[_mouthSmileLeftIndex]; - } - if (_mouthSmileRightIndex != -1) { - _mouthSmileRight = data.m_coeffs[_mouthSmileRightIndex]; - } + _blendshapeCoefficients = data.m_coeffs; } break; } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index c6ab54e694..dc42a4e9c6 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -9,6 +9,8 @@ #ifndef __interface__Faceshift__ #define __interface__Faceshift__ +#include + #include #include @@ -39,20 +41,22 @@ public: float getEstimatedEyePitch() const { return _estimatedEyePitch; } float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } - float getLeftBlink() const { return _leftBlink; } - float getRightBlink() const { return _rightBlink; } - float getLeftEyeOpen() const { return _leftEyeOpen; } - float getRightEyeOpen() const { return _rightEyeOpen; } + const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } - float getBrowDownLeft() const { return _browDownLeft; } - float getBrowDownRight() const { return _browDownRight; } - float getBrowUpCenter() const { return _browUpCenter; } - float getBrowUpLeft() const { return _browUpLeft; } - float getBrowUpRight() const { return _browUpRight; } + float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } + float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } + float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); } + float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); } - float getMouthSize() const { return _mouthSize; } - float getMouthSmileLeft() const { return _mouthSmileLeft; } - float getMouthSmileRight() const { return _mouthSmileRight; } + float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); } + float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); } + float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); } + float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); } + float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); } + + float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); } + float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } + float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } void update(); void reset(); @@ -76,6 +80,8 @@ private slots: private: + float getBlendshapeCoefficient(int index) const; + void send(const std::string& message); void receive(const QByteArray& buffer); @@ -95,35 +101,20 @@ private: float _eyeGazeRightPitch; float _eyeGazeRightYaw; - float _leftBlink; - float _rightBlink; - float _leftEyeOpen; - float _rightEyeOpen; - + std::vector _blendshapeCoefficients; + int _leftBlinkIndex; int _rightBlinkIndex; int _leftEyeOpenIndex; int _rightEyeOpenIndex; // Brows - float _browDownLeft; - float _browDownRight; - float _browUpCenter; - float _browUpLeft; - float _browUpRight; - int _browDownLeftIndex; int _browDownRightIndex; - int _browUpCenterIndex; int _browUpLeftIndex; int _browUpRightIndex; - float _mouthSize; - - float _mouthSmileLeft; - float _mouthSmileRight; - int _mouthSmileLeftIndex; int _mouthSmileRightIndex; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index cb445ffaa7..dab3d1651e 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -204,6 +204,11 @@ int AvatarData::getBroadcastData(unsigned char* destinationBuffer) { memcpy(destinationBuffer, &_headData->_browAudioLift, sizeof(float)); destinationBuffer += sizeof(float); + + *destinationBuffer++ = _headData->_blendshapeCoefficients.size(); + memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), + _headData->_blendshapeCoefficients.size() * sizeof(float)); + destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); } // leap hand data @@ -334,6 +339,11 @@ int AvatarData::parseData(unsigned char* sourceBuffer, int numBytes) { memcpy(&_headData->_browAudioLift, sourceBuffer, sizeof(float)); sourceBuffer += sizeof(float); + + _headData->_blendshapeCoefficients.resize(*sourceBuffer++); + memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, + _headData->_blendshapeCoefficients.size() * sizeof(float)); + sourceBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); } // leap hand data diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index dd4d5b61a4..7855c1eb68 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -10,6 +10,7 @@ #define __hifi__HeadData__ #include +#include #include @@ -52,6 +53,7 @@ public: void setLookAtPosition(const glm::vec3& lookAtPosition) { _lookAtPosition = lookAtPosition; } friend class AvatarData; + protected: float _yaw; float _pitch; @@ -65,7 +67,9 @@ protected: float _rightEyeBlink; float _averageLoudness; float _browAudioLift; + std::vector _blendshapeCoefficients; AvatarData* _owningAvatar; + private: // privatize copy ctor and assignment operator so copies of this object cannot be made HeadData(const HeadData&); diff --git a/libraries/shared/src/PacketHeaders.cpp b/libraries/shared/src/PacketHeaders.cpp index fd261e4385..ff33f35aca 100644 --- a/libraries/shared/src/PacketHeaders.cpp +++ b/libraries/shared/src/PacketHeaders.cpp @@ -20,7 +20,7 @@ PACKET_VERSION versionForPacketType(PACKET_TYPE type) { return 1; case PACKET_TYPE_HEAD_DATA: - return 6; + return 7; case PACKET_TYPE_AVATAR_FACE_VIDEO: return 1; From 354c173c9fbde5e0e2e2e223f927218089552bcd Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Sep 2013 09:58:54 -0700 Subject: [PATCH 03/16] Disable sending coefficients for now. --- libraries/avatars/src/AvatarData.cpp | 4 ++++ libraries/shared/src/PacketHeaders.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index dab3d1651e..4973579243 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -205,10 +205,12 @@ int AvatarData::getBroadcastData(unsigned char* destinationBuffer) { memcpy(destinationBuffer, &_headData->_browAudioLift, sizeof(float)); destinationBuffer += sizeof(float); + /* *destinationBuffer++ = _headData->_blendshapeCoefficients.size(); memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), _headData->_blendshapeCoefficients.size() * sizeof(float)); destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); + */ } // leap hand data @@ -340,10 +342,12 @@ int AvatarData::parseData(unsigned char* sourceBuffer, int numBytes) { memcpy(&_headData->_browAudioLift, sourceBuffer, sizeof(float)); sourceBuffer += sizeof(float); + /* _headData->_blendshapeCoefficients.resize(*sourceBuffer++); memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, _headData->_blendshapeCoefficients.size() * sizeof(float)); sourceBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); + */ } // leap hand data diff --git a/libraries/shared/src/PacketHeaders.cpp b/libraries/shared/src/PacketHeaders.cpp index ff33f35aca..fd261e4385 100644 --- a/libraries/shared/src/PacketHeaders.cpp +++ b/libraries/shared/src/PacketHeaders.cpp @@ -20,7 +20,7 @@ PACKET_VERSION versionForPacketType(PACKET_TYPE type) { return 1; case PACKET_TYPE_HEAD_DATA: - return 7; + return 6; case PACKET_TYPE_AVATAR_FACE_VIDEO: return 1; From 1afece81c575c0ec6c87784794ea948661d0afb0 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Sep 2013 11:49:50 -0700 Subject: [PATCH 04/16] Use quads as well as triangles, apply scale. --- interface/src/avatar/BlendFace.cpp | 18 +++++++++++++----- interface/src/avatar/BlendFace.h | 3 ++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 18c948f8cc..901cdba060 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -34,7 +34,9 @@ bool BlendFace::render(float alpha) { glm::quat orientation = _owningHead->getOrientation(); glm::vec3 axis = glm::axis(orientation); glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); - glScalef(_owningHead->getScale(), _owningHead->getScale(), _owningHead->getScale()); + const float MODEL_SCALE = 0.005f; + glScalef(_owningHead->getScale() * MODEL_SCALE, _owningHead->getScale() * MODEL_SCALE, + _owningHead->getScale() * MODEL_SCALE); glColor4f(1.0f, 1.0f, 1.0f, alpha); @@ -49,7 +51,9 @@ bool BlendFace::render(float alpha) { glPolygonMode(GL_FRONT_AND_BACK, GL_LINE); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); - glDrawRangeElementsEXT(GL_TRIANGLES, 0, _baseVertices.size() - 1, _indexCount, GL_UNSIGNED_INT, 0); + glDrawRangeElementsEXT(GL_QUADS, 0, _baseVertices.size() - 1, _quadIndexCount, GL_UNSIGNED_INT, 0); + glDrawRangeElementsEXT(GL_TRIANGLES, 0, _baseVertices.size() - 1, _triangleIndexCount, GL_UNSIGNED_INT, + (void*)(_quadIndexCount * sizeof(int))); glPolygonMode(GL_FRONT_AND_BACK, GL_FILL); @@ -80,15 +84,19 @@ void BlendFace::setRig(const fsMsgRig& rig) { glGenBuffers(1, &_vboID); } glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); - glBufferData(GL_ELEMENT_ARRAY_BUFFER, rig.mesh().m_tris.size() * sizeof(fsVector3i), - rig.mesh().m_tris.data(), GL_STATIC_DRAW); + glBufferData(GL_ELEMENT_ARRAY_BUFFER, rig.mesh().m_quads.size() * sizeof(fsVector4i) + + rig.mesh().m_tris.size() * sizeof(fsVector3i), NULL, GL_STATIC_DRAW); + glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, 0, rig.mesh().m_quads.size() * sizeof(fsVector4i), rig.mesh().m_quads.data()); + glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, rig.mesh().m_quads.size() * sizeof(fsVector4i), + rig.mesh().m_tris.size() * sizeof(fsVector3i), rig.mesh().m_tris.data()); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); glBindBuffer(GL_ARRAY_BUFFER, _vboID); glBufferData(GL_ARRAY_BUFFER, rig.mesh().m_vertex_data.m_vertices.size() * sizeof(fsVector3f), NULL, GL_DYNAMIC_DRAW); glBindBuffer(GL_ARRAY_BUFFER, 0); - _indexCount = rig.mesh().m_tris.size() * 3; + _quadIndexCount = rig.mesh().m_quads.size() * 4; + _triangleIndexCount = rig.mesh().m_tris.size() * 3; _baseVertices = rig.mesh().m_vertex_data.m_vertices; _blendshapes = rig.blendshapes(); } diff --git a/interface/src/avatar/BlendFace.h b/interface/src/avatar/BlendFace.h index 9385cf4e0b..bcc0689b72 100644 --- a/interface/src/avatar/BlendFace.h +++ b/interface/src/avatar/BlendFace.h @@ -39,7 +39,8 @@ private: GLuint _iboID; GLuint _vboID; - GLsizei _indexCount; + GLsizei _quadIndexCount; + GLsizei _triangleIndexCount; std::vector _baseVertices; std::vector _blendshapes; }; From 558d3d41a6694f4235ede2fb845b88243828c81f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Sep 2013 15:22:06 -0700 Subject: [PATCH 05/16] First attempt at blending. --- interface/src/avatar/BlendFace.cpp | 48 ++++++++++++++++++++++++++++-- interface/src/avatar/BlendFace.h | 3 +- libraries/avatars/src/HeadData.h | 2 ++ 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 901cdba060..692f139dc1 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -6,10 +6,13 @@ // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // +#include + #include "BlendFace.h" #include "Head.h" using namespace fs; +using namespace std; BlendFace::BlendFace(Head* owningHead) : _owningHead(owningHead), @@ -34,15 +37,54 @@ bool BlendFace::render(float alpha) { glm::quat orientation = _owningHead->getOrientation(); glm::vec3 axis = glm::axis(orientation); glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); - const float MODEL_SCALE = 0.005f; + glTranslatef(0.0f, -0.025f, -0.025f); + const float MODEL_SCALE = 0.0006f; glScalef(_owningHead->getScale() * MODEL_SCALE, _owningHead->getScale() * MODEL_SCALE, - _owningHead->getScale() * MODEL_SCALE); + -_owningHead->getScale() * MODEL_SCALE); glColor4f(1.0f, 1.0f, 1.0f, alpha); + // find the coefficient total and scale + const vector& coefficients = _owningHead->getBlendshapeCoefficients(); + float total = accumulate(coefficients.begin(), coefficients.end(), 0.0f); + float scale = 1.0f / (total < 1.0f ? 1.0f : total); + + // start with the base + int vertexCount = _baseVertices.size(); + _blendedVertices.resize(vertexCount); + const fsVector3f* source = _baseVertices.data(); + fsVector3f* dest = _blendedVertices.data(); + float baseCoefficient = (total < 1.0f) ? (1.0f - total) : 0.0f; + for (int i = 0; i < vertexCount; i++) { + dest->x += source->x * baseCoefficient; + dest->y += source->y * baseCoefficient; + dest->z += source->z * baseCoefficient; + + source++; + dest++; + } + + // blend in each coefficient + for (int i = 0; i < coefficients.size(); i++) { + float coefficient = coefficients[i] * scale; + if (coefficient == 0.0f || i >= _blendshapes.size()) { + continue; + } + const fsVector3f* source = _blendshapes[i].m_vertices.data(); + fsVector3f* dest = _blendedVertices.data(); + for (int j = 0; j < vertexCount; j++) { + dest->x += source->x * coefficient; + dest->y += source->y * coefficient; + dest->z += source->z * coefficient; + + source++; + dest++; + } + } + // update the blended vertices glBindBuffer(GL_ARRAY_BUFFER, _vboID); - glBufferSubData(GL_ARRAY_BUFFER, 0, _baseVertices.size() * sizeof(fsVector3f), _baseVertices.data()); + glBufferSubData(GL_ARRAY_BUFFER, 0, _blendedVertices.size() * sizeof(fsVector3f), _blendedVertices.data()); // tell OpenGL where to find vertex information glEnableClientState(GL_VERTEX_ARRAY); diff --git a/interface/src/avatar/BlendFace.h b/interface/src/avatar/BlendFace.h index bcc0689b72..fc13cbfca5 100644 --- a/interface/src/avatar/BlendFace.h +++ b/interface/src/avatar/BlendFace.h @@ -28,7 +28,7 @@ public: bool render(float alpha); -private slots: +public slots: void setRig(const fs::fsMsgRig& rig); @@ -43,6 +43,7 @@ private: GLsizei _triangleIndexCount; std::vector _baseVertices; std::vector _blendshapes; + std::vector _blendedVertices; }; #endif /* defined(__interface__BlendFace__) */ diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 7855c1eb68..88895a58f6 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -44,6 +44,8 @@ public: void setAudioLoudness(float audioLoudness) { _audioLoudness = audioLoudness; } + const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + void addYaw(float yaw); void addPitch(float pitch); void addRoll(float roll); From 617b3d67cf6ff3d2d9549a949bc16f4a34a7277e Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Sep 2013 16:11:07 -0700 Subject: [PATCH 06/16] I think we need to treat the blend shapes as offsets from neutral. --- interface/src/avatar/BlendFace.cpp | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 692f139dc1..dfa4565bab 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -44,29 +44,15 @@ bool BlendFace::render(float alpha) { glColor4f(1.0f, 1.0f, 1.0f, alpha); - // find the coefficient total and scale - const vector& coefficients = _owningHead->getBlendshapeCoefficients(); - float total = accumulate(coefficients.begin(), coefficients.end(), 0.0f); - float scale = 1.0f / (total < 1.0f ? 1.0f : total); - // start with the base int vertexCount = _baseVertices.size(); _blendedVertices.resize(vertexCount); - const fsVector3f* source = _baseVertices.data(); - fsVector3f* dest = _blendedVertices.data(); - float baseCoefficient = (total < 1.0f) ? (1.0f - total) : 0.0f; - for (int i = 0; i < vertexCount; i++) { - dest->x += source->x * baseCoefficient; - dest->y += source->y * baseCoefficient; - dest->z += source->z * baseCoefficient; - - source++; - dest++; - } + memcpy(_blendedVertices.data(), _baseVertices.data(), vertexCount * sizeof(fsVector3f)); // blend in each coefficient + const vector& coefficients = _owningHead->getBlendshapeCoefficients(); for (int i = 0; i < coefficients.size(); i++) { - float coefficient = coefficients[i] * scale; + float coefficient = coefficients[i]; if (coefficient == 0.0f || i >= _blendshapes.size()) { continue; } From 596d0ac471018c6481a1943b97454eb86081e467 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Sep 2013 17:01:17 -0700 Subject: [PATCH 07/16] Subtract the neutral position from the blendshapes. --- interface/src/avatar/BlendFace.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index dfa4565bab..15863e59a9 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -127,4 +127,19 @@ void BlendFace::setRig(const fsMsgRig& rig) { _triangleIndexCount = rig.mesh().m_tris.size() * 3; _baseVertices = rig.mesh().m_vertex_data.m_vertices; _blendshapes = rig.blendshapes(); + + // subtract the neutral locations from the blend shapes; we want the deltas + int vertexCount = _baseVertices.size(); + for (vector::iterator it = _blendshapes.begin(); it != _blendshapes.end(); it++) { + const fsVector3f* neutral = _baseVertices.data(); + fsVector3f* offset = it->m_vertices.data(); + for (int i = 0; i < vertexCount; i++) { + offset->x -= neutral->x; + offset->y -= neutral->y; + offset->z -= neutral->z; + + neutral++; + offset++; + } + } } From 6be03ac3df6c9d4649811bb0656cb77ce1869ac6 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Sep 2013 10:10:24 -0700 Subject: [PATCH 08/16] Working on downloading the faces. --- interface/src/Menu.cpp | 13 ++++++-- interface/src/avatar/Avatar.cpp | 2 ++ interface/src/avatar/BlendFace.cpp | 50 +++++++++++++++++++++++++++++- interface/src/avatar/BlendFace.h | 15 +++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 44bb2e24ff..b0abb4d1cc 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -708,6 +708,10 @@ void Menu::editPreferences() { avatarURL->setMinimumWidth(QLINE_MINIMUM_WIDTH); form->addRow("Avatar URL:", avatarURL); + QLineEdit* faceURL = new QLineEdit(applicationInstance->getAvatar()->getHead().getBlendFace().getModelURL().toString()); + faceURL->setMinimumWidth(QLINE_MINIMUM_WIDTH); + form->addRow("Face URL:", faceURL); + QSpinBox* fieldOfView = new QSpinBox(); fieldOfView->setMaximum(180); fieldOfView->setMinimum(1); @@ -764,9 +768,12 @@ void Menu::editPreferences() { NodeList::getInstance()->setDomainHostname(newHostname.constData()); } - QUrl url(avatarURL->text()); - applicationInstance->getAvatar()->getVoxels()->setVoxelURL(url); - Avatar::sendAvatarVoxelURLMessage(url); + QUrl avatarVoxelURL(avatarURL->text()); + applicationInstance->getAvatar()->getVoxels()->setVoxelURL(avatarVoxelURL); + Avatar::sendAvatarVoxelURLMessage(avatarVoxelURL); + + QUrl faceModelURL(faceURL->text()); + applicationInstance->getAvatar()->getHead().getBlendFace().setModelURL(faceModelURL); _gyroCameraSensitivity = gyroCameraSensitivity->value(); diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 0df2f2d41c..cdfefedaac 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -786,6 +786,7 @@ void Avatar::loadData(QSettings* settings) { _position.z = loadSetting(settings, "position_z", 0.0f); _voxels.setVoxelURL(settings->value("voxelURL").toUrl()); + _head.getBlendFace().setModelURL(settings->value("faceModelURL").toUrl()); _leanScale = loadSetting(settings, "leanScale", 0.05f); @@ -837,6 +838,7 @@ void Avatar::saveData(QSettings* set) { set->setValue("position_z", _position.z); set->setValue("voxelURL", _voxels.getVoxelURL()); + set->setValue("faceModelURL", _head.getBlendFace().getModelURL()); set->setValue("leanScale", _leanScale); set->setValue("scale", _newScale); diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 15863e59a9..ef3e6ff6af 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -6,8 +6,9 @@ // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // -#include +#include +#include "Application.h" #include "BlendFace.h" #include "Head.h" @@ -16,6 +17,7 @@ using namespace std; BlendFace::BlendFace(Head* owningHead) : _owningHead(owningHead), + _modelReply(NULL), _iboID(0) { } @@ -97,6 +99,30 @@ bool BlendFace::render(float alpha) { return true; } +void BlendFace::setModelURL(const QUrl& url) { + // don't restart the download if it's the same URL + if (_modelURL == url) { + return; + } + + // cancel any current download + if (_modelReply != 0) { + delete _modelReply; + _modelReply = 0; + } + + // remember the URL + _modelURL = url; + + // load the URL data asynchronously + if (!url.isValid()) { + return; + } + _modelReply = Application::getInstance()->getNetworkAccessManager()->get(QNetworkRequest(url)); + connect(_modelReply, SIGNAL(downloadProgress(qint64,qint64)), SLOT(handleModelDownloadProgress(qint64,qint64))); + connect(_modelReply, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(handleModelReplyError())); +} + void BlendFace::setRig(const fsMsgRig& rig) { if (rig.mesh().m_tris.empty()) { // clear any existing geometry @@ -143,3 +169,25 @@ void BlendFace::setRig(const fsMsgRig& rig) { } } } + +void BlendFace::handleModelDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { + if (bytesReceived < bytesTotal) { + return; + } + + QByteArray entirety = _modelReply->readAll(); + _modelReply->disconnect(this); + _modelReply->deleteLater(); + _modelReply = 0; + + qDebug("Got %d bytes.\n", entirety.size()); +} + +void BlendFace::handleModelReplyError() { + qDebug("%s\n", _modelReply->errorString().toLocal8Bit().constData()); + + _modelReply->disconnect(this); + _modelReply->deleteLater(); + _modelReply = 0; +} + diff --git a/interface/src/avatar/BlendFace.h b/interface/src/avatar/BlendFace.h index fc13cbfca5..184b31a236 100644 --- a/interface/src/avatar/BlendFace.h +++ b/interface/src/avatar/BlendFace.h @@ -10,11 +10,14 @@ #define __interface__BlendFace__ #include +#include #include #include "InterfaceConfig.h" +class QNetworkReply; + class Head; /// A face formed from a linear mix of blendshapes according to a set of coefficients. @@ -28,14 +31,26 @@ public: bool render(float alpha); + Q_INVOKABLE void setModelURL(const QUrl& url); + const QUrl& getModelURL() const { return _modelURL; } + public slots: void setRig(const fs::fsMsgRig& rig); + +private slots: + + void handleModelDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); + void handleModelReplyError(); private: Head* _owningHead; + QUrl _modelURL; + + QNetworkReply* _modelReply; + GLuint _iboID; GLuint _vboID; From dfea69ab8f664aca87a88fc93643d3b96930c147 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Sep 2013 16:26:35 -0700 Subject: [PATCH 09/16] Basic FBX parser. --- interface/src/avatar/BlendFace.cpp | 8 +- interface/src/renderer/FBXReader.cpp | 192 +++++++++++++++++++++++++++ interface/src/renderer/FBXReader.h | 40 ++++++ 3 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 interface/src/renderer/FBXReader.cpp create mode 100644 interface/src/renderer/FBXReader.h diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index ef3e6ff6af..7e30cfdcda 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -11,6 +11,7 @@ #include "Application.h" #include "BlendFace.h" #include "Head.h" +#include "renderer/FBXReader.h" using namespace fs; using namespace std; @@ -180,7 +181,12 @@ void BlendFace::handleModelDownloadProgress(qint64 bytesReceived, qint64 bytesTo _modelReply->deleteLater(); _modelReply = 0; - qDebug("Got %d bytes.\n", entirety.size()); + try { + printNode(parseFBX(entirety)); + + } catch (const QString& error) { + qDebug() << error << "\n"; + } } void BlendFace::handleModelReplyError() { diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp new file mode 100644 index 0000000000..385ed17d4d --- /dev/null +++ b/interface/src/renderer/FBXReader.cpp @@ -0,0 +1,192 @@ +// +// FBXReader.cpp +// interface +// +// Created by Andrzej Kapolka on 9/18/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include +#include +#include +#include +#include + +#include "FBXReader.h" + +FBXNode parseFBX(const QByteArray& data) { + QBuffer buffer(const_cast(&data)); + buffer.open(QIODevice::ReadOnly); + return parseFBX(&buffer); +} + +template QVariant readArray(QDataStream& in) { + quint32 arrayLength; + quint32 encoding; + quint32 compressedLength; + + in >> arrayLength; + in >> encoding; + in >> compressedLength; + + QVector values; + const int DEFLATE_ENCODING = 1; + if (encoding == DEFLATE_ENCODING) { + // preface encoded data with uncompressed length + QByteArray compressed(sizeof(quint32) + compressedLength, 0); + *((quint32*)compressed.data()) = qToBigEndian(arrayLength * sizeof(T)); + in.readRawData(compressed.data() + sizeof(quint32), compressedLength); + QByteArray uncompressed = qUncompress(compressed); + QDataStream uncompressedIn(uncompressed); + uncompressedIn.setByteOrder(QDataStream::LittleEndian); + for (int i = 0; i < arrayLength; i++) { + T value; + uncompressedIn >> value; + values.append(value); + } + } else { + for (int i = 0; i < arrayLength; i++) { + T value; + in >> value; + values.append(value); + } + } + return QVariant::fromValue(values); +} + +QVariant parseFBXProperty(QDataStream& in) { + char ch; + in.device()->getChar(&ch); + switch (ch) { + case 'Y': { + qint16 value; + in >> value; + return QVariant::fromValue(value); + } + case 'C': { + bool value; + in >> value; + return QVariant::fromValue(value); + } + case 'I': { + qint32 value; + in >> value; + return QVariant::fromValue(value); + } + case 'F': { + float value; + in >> value; + return QVariant::fromValue(value); + } + case 'D': { + double value; + in >> value; + return QVariant::fromValue(value); + } + case 'L': { + qint64 value; + in >> value; + return QVariant::fromValue(value); + } + case 'f': { + return readArray(in); + } + case 'd': { + return readArray(in); + } + case 'l': { + return readArray(in); + } + case 'i': { + return readArray(in); + } + case 'b': { + return readArray(in); + } + case 'S': + case 'R': { + quint32 length; + in >> length; + return QVariant::fromValue(in.device()->read(length)); + } + default: + throw QString("Unknown property type: ") + ch; + } +} + +FBXNode parseFBXNode(QDataStream& in) { + quint32 endOffset; + quint32 propertyCount; + quint32 propertyListLength; + quint8 nameLength; + + in >> endOffset; + in >> propertyCount; + in >> propertyListLength; + in >> nameLength; + + FBXNode node; + const int MIN_VALID_OFFSET = 40; + if (endOffset < MIN_VALID_OFFSET || nameLength == 0) { + // use a null name to indicate a null node + return node; + } + node.name = in.device()->read(nameLength); + + for (int i = 0; i < propertyCount; i++) { + node.properties.append(parseFBXProperty(in)); + } + + while (endOffset > in.device()->pos()) { + FBXNode child = parseFBXNode(in); + if (child.name.isNull()) { + return node; + + } else { + node.children.append(child); + } + } + + return node; +} + +FBXNode parseFBX(QIODevice* device) { + QDataStream in(device); + in.setByteOrder(QDataStream::LittleEndian); + + // verify the prolog + const QByteArray EXPECTED_PROLOG = "Kaydara FBX Binary "; + if (device->read(EXPECTED_PROLOG.size()) != EXPECTED_PROLOG) { + throw QString("Invalid header."); + } + + // skip the rest of the header + const int HEADER_SIZE = 27; + in.skipRawData(HEADER_SIZE - EXPECTED_PROLOG.size()); + + // parse the top-level node + FBXNode top; + while (device->bytesAvailable() >= 13) { + FBXNode next = parseFBXNode(in); + if (next.name.isNull()) { + return top; + + } else { + top.children.append(next); + } + } + + return top; +} + +void printNode(const FBXNode& node, int indent) { + QByteArray spaces(indent, ' '); + qDebug("%s%s: ", spaces.data(), node.name.data()); + foreach (const QVariant& property, node.properties) { + qDebug() << property; + } + qDebug() << "\n"; + foreach (const FBXNode& child, node.children) { + printNode(child, indent + 1); + } +} diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h new file mode 100644 index 0000000000..92fa293560 --- /dev/null +++ b/interface/src/renderer/FBXReader.h @@ -0,0 +1,40 @@ +// +// FBXReader.h +// interface +// +// Created by Andrzej Kapolka on 9/18/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__FBXReader__ +#define __interface__FBXReader__ + +#include +#include + +class QIODevice; + +class FBXNode; + +typedef QList FBXNodeList; + +/// A node within an FBX document. +class FBXNode { +public: + + QByteArray name; + QVariantList properties; + FBXNodeList children; +}; + +/// Parses the input from the supplied data as an FBX file. +/// \exception QString if an error occurs in parsing +FBXNode parseFBX(const QByteArray& data); + +/// Parses the input from the supplied device as an FBX file. +/// \exception QString if an error occurs in parsing +FBXNode parseFBX(QIODevice* device); + +void printNode(const FBXNode& node, int indent = 0); + +#endif /* defined(__interface__FBXReader__) */ From 09fa782c616d8564d107438a5724fcefac444f33 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 19 Sep 2013 15:05:17 -0700 Subject: [PATCH 10/16] More work on FBX reading. --- assignment-client/src/avatars/AvatarMixer.cpp | 3 +- interface/src/avatar/BlendFace.cpp | 147 +++++++++++------- interface/src/avatar/BlendFace.h | 10 +- interface/src/renderer/FBXReader.cpp | 104 +++++++++++++ interface/src/renderer/FBXReader.h | 26 ++++ libraries/shared/src/PacketHeaders.h | 1 + 6 files changed, 226 insertions(+), 65 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 49c0b02d0c..ed7bf91d70 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -139,6 +139,7 @@ void AvatarMixer::run() { break; case PACKET_TYPE_AVATAR_VOXEL_URL: case PACKET_TYPE_AVATAR_FACE_VIDEO: + case PACKET_TYPE_AVATAR_FACE_URL: // grab the node ID from the packet unpackNodeId(packetData + numBytesForPacketHeader(packetData), &nodeID); @@ -158,4 +159,4 @@ void AvatarMixer::run() { } nodeList->stopSilentNodeRemovalThread(); -} \ No newline at end of file +} diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 7e30cfdcda..71988a1a41 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -11,7 +11,6 @@ #include "Application.h" #include "BlendFace.h" #include "Head.h" -#include "renderer/FBXReader.h" using namespace fs; using namespace std; @@ -48,32 +47,27 @@ bool BlendFace::render(float alpha) { glColor4f(1.0f, 1.0f, 1.0f, alpha); // start with the base - int vertexCount = _baseVertices.size(); + int vertexCount = _geometry.vertices.size(); _blendedVertices.resize(vertexCount); - memcpy(_blendedVertices.data(), _baseVertices.data(), vertexCount * sizeof(fsVector3f)); + memcpy(_blendedVertices.data(), _geometry.vertices.constData(), vertexCount * sizeof(glm::vec3)); // blend in each coefficient const vector& coefficients = _owningHead->getBlendshapeCoefficients(); for (int i = 0; i < coefficients.size(); i++) { float coefficient = coefficients[i]; - if (coefficient == 0.0f || i >= _blendshapes.size()) { + if (coefficient == 0.0f || i >= _geometry.blendshapes.size()) { continue; } - const fsVector3f* source = _blendshapes[i].m_vertices.data(); - fsVector3f* dest = _blendedVertices.data(); - for (int j = 0; j < vertexCount; j++) { - dest->x += source->x * coefficient; - dest->y += source->y * coefficient; - dest->z += source->z * coefficient; - - source++; - dest++; + const glm::vec3* source = _geometry.blendshapes[i].vertices.constData(); + for (const int* index = _geometry.blendshapes[i].indices.constData(), + *end = index + _geometry.blendshapes[i].indices.size(); index != end; index++, source++) { + _blendedVertices[*index] += *source * coefficient; } } // update the blended vertices glBindBuffer(GL_ARRAY_BUFFER, _vboID); - glBufferSubData(GL_ARRAY_BUFFER, 0, _blendedVertices.size() * sizeof(fsVector3f), _blendedVertices.data()); + glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), _blendedVertices.constData()); // tell OpenGL where to find vertex information glEnableClientState(GL_VERTEX_ARRAY); @@ -82,9 +76,9 @@ bool BlendFace::render(float alpha) { glPolygonMode(GL_FRONT_AND_BACK, GL_LINE); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); - glDrawRangeElementsEXT(GL_QUADS, 0, _baseVertices.size() - 1, _quadIndexCount, GL_UNSIGNED_INT, 0); - glDrawRangeElementsEXT(GL_TRIANGLES, 0, _baseVertices.size() - 1, _triangleIndexCount, GL_UNSIGNED_INT, - (void*)(_quadIndexCount * sizeof(int))); + glDrawRangeElementsEXT(GL_QUADS, 0, vertexCount - 1, _geometry.quadIndices.size(), GL_UNSIGNED_INT, 0); + glDrawRangeElementsEXT(GL_TRIANGLES, 0, vertexCount - 1, _geometry.triangleIndices.size(), GL_UNSIGNED_INT, + (void*)(_geometry.quadIndices.size() * sizeof(int))); glPolygonMode(GL_FRONT_AND_BACK, GL_FILL); @@ -124,51 +118,42 @@ void BlendFace::setModelURL(const QUrl& url) { connect(_modelReply, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(handleModelReplyError())); } +glm::vec3 createVec3(const fsVector3f& vector) { + return glm::vec3(vector.x, vector.y, vector.z); +} + void BlendFace::setRig(const fsMsgRig& rig) { - if (rig.mesh().m_tris.empty()) { - // clear any existing geometry - if (_iboID != 0) { - glDeleteBuffers(1, &_iboID); - glDeleteBuffers(1, &_vboID); - _iboID = 0; + // convert to FBX geometry + FBXGeometry geometry; + + for (vector::const_iterator it = rig.mesh().m_quads.begin(), end = rig.mesh().m_quads.end(); it != end; it++) { + geometry.quadIndices.append(it->x); + geometry.quadIndices.append(it->y); + geometry.quadIndices.append(it->z); + geometry.quadIndices.append(it->w); + } + + for (vector::const_iterator it = rig.mesh().m_tris.begin(), end = rig.mesh().m_tris.end(); it != end; it++) { + geometry.triangleIndices.append(it->x); + geometry.triangleIndices.append(it->y); + geometry.triangleIndices.append(it->z); + } + + for (vector::const_iterator it = rig.mesh().m_vertex_data.m_vertices.begin(), + end = rig.mesh().m_vertex_data.m_vertices.end(); it != end; it++) { + geometry.vertices.append(glm::vec3(it->x, it->y, it->z)); + } + + for (vector::const_iterator it = rig.blendshapes().begin(), end = rig.blendshapes().end(); it != end; it++) { + FBXBlendshape blendshape; + for (int i = 0, n = it->m_vertices.size(); i < n; i++) { + blendshape.indices.append(i); + blendshape.vertices.append(createVec3(it->m_vertices[i])); } - return; + geometry.blendshapes.append(blendshape); } - if (_iboID == 0) { - glGenBuffers(1, &_iboID); - glGenBuffers(1, &_vboID); - } - glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); - glBufferData(GL_ELEMENT_ARRAY_BUFFER, rig.mesh().m_quads.size() * sizeof(fsVector4i) + - rig.mesh().m_tris.size() * sizeof(fsVector3i), NULL, GL_STATIC_DRAW); - glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, 0, rig.mesh().m_quads.size() * sizeof(fsVector4i), rig.mesh().m_quads.data()); - glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, rig.mesh().m_quads.size() * sizeof(fsVector4i), - rig.mesh().m_tris.size() * sizeof(fsVector3i), rig.mesh().m_tris.data()); - glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); - glBindBuffer(GL_ARRAY_BUFFER, _vboID); - glBufferData(GL_ARRAY_BUFFER, rig.mesh().m_vertex_data.m_vertices.size() * sizeof(fsVector3f), NULL, GL_DYNAMIC_DRAW); - glBindBuffer(GL_ARRAY_BUFFER, 0); - - _quadIndexCount = rig.mesh().m_quads.size() * 4; - _triangleIndexCount = rig.mesh().m_tris.size() * 3; - _baseVertices = rig.mesh().m_vertex_data.m_vertices; - _blendshapes = rig.blendshapes(); - - // subtract the neutral locations from the blend shapes; we want the deltas - int vertexCount = _baseVertices.size(); - for (vector::iterator it = _blendshapes.begin(); it != _blendshapes.end(); it++) { - const fsVector3f* neutral = _baseVertices.data(); - fsVector3f* offset = it->m_vertices.data(); - for (int i = 0; i < vertexCount; i++) { - offset->x -= neutral->x; - offset->y -= neutral->y; - offset->z -= neutral->z; - - neutral++; - offset++; - } - } + setGeometry(geometry); } void BlendFace::handleModelDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { @@ -182,10 +167,11 @@ void BlendFace::handleModelDownloadProgress(qint64 bytesReceived, qint64 bytesTo _modelReply = 0; try { - printNode(parseFBX(entirety)); + setGeometry(extractFBXGeometry(parseFBX(entirety))); } catch (const QString& error) { qDebug() << error << "\n"; + return; } } @@ -197,3 +183,46 @@ void BlendFace::handleModelReplyError() { _modelReply = 0; } +void BlendFace::setGeometry(const FBXGeometry& geometry) { + if (geometry.vertices.isEmpty()) { + // clear any existing geometry + if (_iboID != 0) { + glDeleteBuffers(1, &_iboID); + glDeleteBuffers(1, &_vboID); + _iboID = 0; + } + return; + } + if (_iboID == 0) { + glGenBuffers(1, &_iboID); + glGenBuffers(1, &_vboID); + } + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); + glBufferData(GL_ELEMENT_ARRAY_BUFFER, (geometry.quadIndices.size() + geometry.triangleIndices.size()) * sizeof(int), + NULL, GL_STATIC_DRAW); + glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, 0, geometry.quadIndices.size() * sizeof(int), geometry.quadIndices.constData()); + glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, geometry.quadIndices.size() * sizeof(int), + geometry.triangleIndices.size() * sizeof(int), geometry.triangleIndices.constData()); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + + glBindBuffer(GL_ARRAY_BUFFER, _vboID); + glBufferData(GL_ARRAY_BUFFER, geometry.vertices.size() * sizeof(glm::vec3), NULL, GL_DYNAMIC_DRAW); + glBindBuffer(GL_ARRAY_BUFFER, 0); + + _geometry = geometry; + + // subtract the neutral locations from the blend shapes; we want the deltas + /* + int vertexCount = geometry.vertices.size(); + for (QVector::iterator it = _geometry.blendshapes.begin(); it != _geometry.blendshapes.end(); it++) { + const glm::vec3* neutral = geometry.vertices.constData(); + glm::vec3* offset = it->vertices.data(); + for (int i = 0; i < vertexCount; i++) { + *offset -= *neutral; + + neutral++; + offset++; + } + } + */ +} diff --git a/interface/src/avatar/BlendFace.h b/interface/src/avatar/BlendFace.h index 184b31a236..937d696edf 100644 --- a/interface/src/avatar/BlendFace.h +++ b/interface/src/avatar/BlendFace.h @@ -15,6 +15,7 @@ #include #include "InterfaceConfig.h" +#include "renderer/FBXReader.h" class QNetworkReply; @@ -45,6 +46,8 @@ private slots: private: + void setGeometry(const FBXGeometry& geometry); + Head* _owningHead; QUrl _modelURL; @@ -54,11 +57,8 @@ private: GLuint _iboID; GLuint _vboID; - GLsizei _quadIndexCount; - GLsizei _triangleIndexCount; - std::vector _baseVertices; - std::vector _blendshapes; - std::vector _blendedVertices; + FBXGeometry _geometry; + QVector _blendedVertices; }; #endif /* defined(__interface__BlendFace__) */ diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 385ed17d4d..8c7350e790 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -14,6 +14,8 @@ #include "FBXReader.h" +using namespace std; + FBXNode parseFBX(const QByteArray& data) { QBuffer buffer(const_cast(&data)); buffer.open(QIODevice::ReadOnly); @@ -154,6 +156,9 @@ FBXNode parseFBX(QIODevice* device) { QDataStream in(device); in.setByteOrder(QDataStream::LittleEndian); + // see http://code.blender.org/index.php/2013/08/fbx-binary-file-format-specification/ for an explanation + // of the FBX format + // verify the prolog const QByteArray EXPECTED_PROLOG = "Kaydara FBX Binary "; if (device->read(EXPECTED_PROLOG.size()) != EXPECTED_PROLOG) { @@ -179,6 +184,104 @@ FBXNode parseFBX(QIODevice* device) { return top; } +QVector createVec3Vector(const QVector& doubleVector) { + QVector values; + for (const double* it = doubleVector.constData(), *end = it + doubleVector.size(); it != end; ) { + values.append(glm::vec3(*it++, *it++, *it++)); + } + return values; +} + +FBXGeometry extractFBXGeometry(const FBXNode& node) { + FBXGeometry geometry; + + // "Objects" + // "Geometry" QVariant(qlonglong, 5386006736) QVariant(QByteArray, "QVariant(QByteArray, "Mesh") + // "Vertices" QVector + // "PolygonVertexIndex" QVector + // "LayerElementNormal" + // "Normals" QVector + // "Geometry" QVariant(qlonglong, 5470612480) QVariant(QByteArray, "BrowsD_LQVariant(QByteArray, "Shape") + // "Vertices" QVector + // "Normals" QVector + + foreach (const FBXNode& child, node.children) { + if (child.name == "Objects") { + foreach (const FBXNode& object, child.children) { + if (object.name == "Geometry") { + if (object.properties.at(2) == "Mesh") { + QVector vertices; + QVector polygonIndices; + foreach (const FBXNode& data, object.children) { + if (data.name == "Vertices") { + geometry.vertices = createVec3Vector(data.properties.at(0).value >()); + + } else if (data.name == "PolygonVertexIndex") { + polygonIndices = data.properties.at(0).value >(); + + } else if (data.name == "LayerElementNormal") { + foreach (const FBXNode& subdata, data.children) { + if (subdata.name == "Normals") { + geometry.normals = createVec3Vector( + subdata.properties.at(0).value >()); + } + } + } + } + // convert the polygons to quads and triangles + for (const int* beginIndex = polygonIndices.constData(), *end = beginIndex + polygonIndices.size(); + beginIndex != end; ) { + const int* endIndex = beginIndex; + while (*endIndex++ >= 0); + + if (endIndex - beginIndex == 4) { + geometry.quadIndices.append(*beginIndex++); + geometry.quadIndices.append(*beginIndex++); + geometry.quadIndices.append(*beginIndex++); + geometry.quadIndices.append(-*beginIndex++ - 1); + + } else { + for (const int* nextIndex = beginIndex + 1;; ) { + geometry.triangleIndices.append(*beginIndex); + geometry.triangleIndices.append(*nextIndex++); + if (*nextIndex >= 0) { + geometry.triangleIndices.append(*nextIndex); + } else { + geometry.triangleIndices.append(-*nextIndex - 1); + break; + } + } + beginIndex = endIndex; + } + } + + } else { // object.properties.at(2) == "Shape" + FBXBlendshape blendshape; + foreach (const FBXNode& data, object.children) { + if (data.name == "Indexes") { + blendshape.indices = data.properties.at(0).value >(); + + } else if (data.name == "Vertices") { + blendshape.vertices = createVec3Vector(data.properties.at(0).value >()); + + } else if (data.name == "Normals") { + blendshape.normals = createVec3Vector(data.properties.at(0).value >()); + } + } + + // the name is followed by a null and some type info + QByteArray name = object.properties.at(1).toByteArray(); + name = name.left(name.indexOf('\0')); + geometry.blendshapes.append(blendshape); + } + } + } + } + } + + return geometry; +} + void printNode(const FBXNode& node, int indent) { QByteArray spaces(indent, ' '); qDebug("%s%s: ", spaces.data(), node.name.data()); @@ -190,3 +293,4 @@ void printNode(const FBXNode& node, int indent) { printNode(child, indent + 1); } } + diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 92fa293560..52447d20b4 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -12,6 +12,8 @@ #include #include +#include + class QIODevice; class FBXNode; @@ -27,6 +29,27 @@ public: FBXNodeList children; }; +/// A single blendshape extracted from an FBX document. +class FBXBlendshape { +public: + + QVector indices; + QVector vertices; + QVector normals; +}; + +/// Base geometry with blendshapes mapped by name. +class FBXGeometry { +public: + + QVector quadIndices; + QVector triangleIndices; + QVector vertices; + QVector normals; + + QVector blendshapes; +}; + /// Parses the input from the supplied data as an FBX file. /// \exception QString if an error occurs in parsing FBXNode parseFBX(const QByteArray& data); @@ -35,6 +58,9 @@ FBXNode parseFBX(const QByteArray& data); /// \exception QString if an error occurs in parsing FBXNode parseFBX(QIODevice* device); +/// Extracts the geometry from a parsed FBX node. +FBXGeometry extractFBXGeometry(const FBXNode& node); + void printNode(const FBXNode& node, int indent = 0); #endif /* defined(__interface__FBXReader__) */ diff --git a/libraries/shared/src/PacketHeaders.h b/libraries/shared/src/PacketHeaders.h index 10ddbf56ad..59379d5dd8 100644 --- a/libraries/shared/src/PacketHeaders.h +++ b/libraries/shared/src/PacketHeaders.h @@ -31,6 +31,7 @@ const PACKET_TYPE PACKET_TYPE_VOXEL_DATA_MONOCHROME = 'v'; const PACKET_TYPE PACKET_TYPE_BULK_AVATAR_DATA = 'X'; const PACKET_TYPE PACKET_TYPE_AVATAR_VOXEL_URL = 'U'; const PACKET_TYPE PACKET_TYPE_AVATAR_FACE_VIDEO = 'F'; +const PACKET_TYPE PACKET_TYPE_AVATAR_FACE_URL = 'f'; const PACKET_TYPE PACKET_TYPE_TRANSMITTER_DATA_V2 = 'T'; const PACKET_TYPE PACKET_TYPE_ENVIRONMENT_DATA = 'e'; const PACKET_TYPE PACKET_TYPE_DOMAIN_LIST_REQUEST = 'L'; From 116a4e0bad65fc982b7a31677a53cf6c8c14661d Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 19 Sep 2013 15:18:45 -0700 Subject: [PATCH 11/16] Subtract the deltas. --- interface/src/avatar/BlendFace.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 71988a1a41..8280e8b7df 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -212,17 +212,10 @@ void BlendFace::setGeometry(const FBXGeometry& geometry) { _geometry = geometry; // subtract the neutral locations from the blend shapes; we want the deltas - /* - int vertexCount = geometry.vertices.size(); for (QVector::iterator it = _geometry.blendshapes.begin(); it != _geometry.blendshapes.end(); it++) { - const glm::vec3* neutral = geometry.vertices.constData(); glm::vec3* offset = it->vertices.data(); - for (int i = 0; i < vertexCount; i++) { - *offset -= *neutral; - - neutral++; - offset++; + for (const int* index = it->indices.constData(), *end = index + it->indices.size(); index != end; index++, offset++) { + *offset -= geometry.vertices[*index]; } } - */ } From fff10375a90a573618d82259bd8984a5cea3ade3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 19 Sep 2013 15:48:33 -0700 Subject: [PATCH 12/16] Map the blendshapes by name. --- interface/src/avatar/BlendFace.cpp | 13 ++--- interface/src/renderer/FBXReader.cpp | 71 +++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 8280e8b7df..690a278453 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -55,7 +55,7 @@ bool BlendFace::render(float alpha) { const vector& coefficients = _owningHead->getBlendshapeCoefficients(); for (int i = 0; i < coefficients.size(); i++) { float coefficient = coefficients[i]; - if (coefficient == 0.0f || i >= _geometry.blendshapes.size()) { + if (coefficient == 0.0f || i >= _geometry.blendshapes.size() || _geometry.blendshapes[i].vertices.isEmpty()) { continue; } const glm::vec3* source = _geometry.blendshapes[i].vertices.constData(); @@ -147,8 +147,9 @@ void BlendFace::setRig(const fsMsgRig& rig) { for (vector::const_iterator it = rig.blendshapes().begin(), end = rig.blendshapes().end(); it != end; it++) { FBXBlendshape blendshape; for (int i = 0, n = it->m_vertices.size(); i < n; i++) { + // subtract the base vertex position; we want the deltas + blendshape.vertices.append(createVec3(it->m_vertices[i]) - geometry.vertices[i]); blendshape.indices.append(i); - blendshape.vertices.append(createVec3(it->m_vertices[i])); } geometry.blendshapes.append(blendshape); } @@ -210,12 +211,4 @@ void BlendFace::setGeometry(const FBXGeometry& geometry) { glBindBuffer(GL_ARRAY_BUFFER, 0); _geometry = geometry; - - // subtract the neutral locations from the blend shapes; we want the deltas - for (QVector::iterator it = _geometry.blendshapes.begin(); it != _geometry.blendshapes.end(); it++) { - glm::vec3* offset = it->vertices.data(); - for (const int* index = it->indices.constData(), *end = index + it->indices.size(); index != end; index++, offset++) { - *offset -= geometry.vertices[*index]; - } - } } diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 8c7350e790..ce4dc35934 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -192,6 +192,71 @@ QVector createVec3Vector(const QVector& doubleVector) { return values; } +const char* FACESHIFT_BLENDSHAPES[] = { + "EyeBlink_L", + "EyeBlink_R", + "EyeSquint_L", + "EyeSquint_R", + "EyeDown_L", + "EyeDown_R", + "EyeIn_L", + "EyeIn_R", + "EyeOpen_L", + "EyeOpen_R", + "EyeOut_L", + "EyeOut_R", + "EyeUp_L", + "EyeUp_R", + "BrowsD_L", + "BrowsD_R", + "BrowsU_C", + "BrowsU_L", + "BrowsU_R", + "JawFwd", + "JawLeft", + "JawOpen", + "JawChew", + "JawRight", + "MouthLeft", + "MouthRight", + "MouthFrown_L", + "MouthFrown_R", + "MouthSmile_L", + "MouthSmile_R", + "MouthDimple_L", + "MouthDimple_R", + "LipsStretch_L", + "LipsStretch_R", + "LipsUpperClose", + "LipsLowerClose", + "LipsUpperUp", + "LipsLowerDown", + "LipsUpperOpen", + "LipsLowerOpen", + "LipsFunnel", + "LipsPucker", + "ChinLowerRaise", + "ChinUpperRaise", + "Sneer", + "Puff", + "CheekSquint_L", + "CheekSquint_R", + "" +}; + +QHash createBlendshapeMap() { + QHash map; + for (int i = 0;; i++) { + QByteArray name = FACESHIFT_BLENDSHAPES[i]; + if (name != "") { + map.insert(name, i); + + } else { + return map; + } + } +} + FBXGeometry extractFBXGeometry(const FBXNode& node) { FBXGeometry geometry; @@ -271,8 +336,10 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { // the name is followed by a null and some type info QByteArray name = object.properties.at(1).toByteArray(); - name = name.left(name.indexOf('\0')); - geometry.blendshapes.append(blendshape); + static QHash blendshapeMap = createBlendshapeMap(); + int index = blendshapeMap.value(name.left(name.indexOf('\0'))); + geometry.blendshapes.resize(qMax(geometry.blendshapes.size(), index + 1)); + geometry.blendshapes[index] = blendshape; } } } From a04ced633ebb4341b31246bb3a82b9f939b399bb Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 19 Sep 2013 16:35:25 -0700 Subject: [PATCH 13/16] Include the face model URL in the same packet as the avatar voxel URL. --- assignment-client/src/avatars/AvatarMixer.cpp | 3 +- interface/src/Application.cpp | 31 +++++++++++-------- interface/src/Application.h | 2 +- interface/src/Menu.cpp | 3 +- interface/src/avatar/Avatar.cpp | 9 ++++-- interface/src/avatar/Avatar.h | 2 +- libraries/avatars/src/AvatarData.cpp | 4 --- libraries/shared/src/PacketHeaders.cpp | 5 ++- libraries/shared/src/PacketHeaders.h | 3 +- 9 files changed, 34 insertions(+), 28 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index ed7bf91d70..b074981a12 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -137,9 +137,8 @@ void AvatarMixer::run() { case PACKET_TYPE_INJECT_AUDIO: broadcastAvatarData(nodeList, nodeAddress); break; - case PACKET_TYPE_AVATAR_VOXEL_URL: + case PACKET_TYPE_AVATAR_URLS: case PACKET_TYPE_AVATAR_FACE_VIDEO: - case PACKET_TYPE_AVATAR_FACE_URL: // grab the node ID from the packet unpackNodeId(packetData + numBytesForPacketHeader(packetData), &nodeID); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 58bb48cae6..7be8582cdc 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1230,15 +1230,19 @@ static Avatar* processAvatarMessageHeader(unsigned char*& packetData, size_t& da return avatar->isInitialized() ? avatar : NULL; } -void Application::processAvatarVoxelURLMessage(unsigned char* packetData, size_t dataBytes) { +void Application::processAvatarURLsMessage(unsigned char* packetData, size_t dataBytes) { Avatar* avatar = processAvatarMessageHeader(packetData, dataBytes); if (!avatar) { return; - } - QUrl url = QUrl::fromEncoded(QByteArray((char*)packetData, dataBytes)); + } + QDataStream in(QByteArray((char*)packetData, dataBytes)); + QUrl voxelURL, faceURL; + in >> voxelURL; + in >> faceURL; - // invoke the set URL function on the simulate/render thread - QMetaObject::invokeMethod(avatar->getVoxels(), "setVoxelURL", Q_ARG(QUrl, url)); + // invoke the set URL functions on the simulate/render thread + QMetaObject::invokeMethod(avatar->getVoxels(), "setVoxelURL", Q_ARG(QUrl, voxelURL)); + QMetaObject::invokeMethod(&avatar->getHead().getBlendFace(), "setModelURL", Q_ARG(QUrl, faceURL)); } void Application::processAvatarFaceVideoMessage(unsigned char* packetData, size_t dataBytes) { @@ -1564,8 +1568,8 @@ void Application::init() { qDebug("Loaded settings.\n"); - Avatar::sendAvatarVoxelURLMessage(_myAvatar.getVoxels()->getVoxelURL()); - + Avatar::sendAvatarURLsMessage(_myAvatar.getVoxels()->getVoxelURL(), _myAvatar.getHead().getBlendFace().getModelURL()); + _palette.init(_glWidget->width(), _glWidget->height()); _palette.addAction(Menu::getInstance()->getActionForOption(MenuOption::VoxelAddMode), 0, 0); _palette.addAction(Menu::getInstance()->getActionForOption(MenuOption::VoxelDeleteMode), 0, 1); @@ -2136,10 +2140,11 @@ void Application::updateAvatar(float deltaTime) { controlledBroadcastToNodes(broadcastString, endOfBroadcastStringWrite - broadcastString, nodeTypesOfInterest, sizeof(nodeTypesOfInterest)); - // once in a while, send my voxel url - const float AVATAR_VOXEL_URL_SEND_INTERVAL = 1.0f; // seconds - if (shouldDo(AVATAR_VOXEL_URL_SEND_INTERVAL, deltaTime)) { - Avatar::sendAvatarVoxelURLMessage(_myAvatar.getVoxels()->getVoxelURL()); + // once in a while, send my urls + const float AVATAR_URLS_SEND_INTERVAL = 1.0f; // seconds + if (shouldDo(AVATAR_URLS_SEND_INTERVAL, deltaTime)) { + Avatar::sendAvatarURLsMessage(_myAvatar.getVoxels()->getVoxelURL(), + _myAvatar.getHead().getBlendFace().getModelURL()); } } } @@ -3510,8 +3515,8 @@ void* Application::networkReceive(void* args) { bytesReceived); getInstance()->_bandwidthMeter.inputStream(BandwidthMeter::AVATARS).updateValue(bytesReceived); break; - case PACKET_TYPE_AVATAR_VOXEL_URL: - processAvatarVoxelURLMessage(app->_incomingPacket, bytesReceived); + case PACKET_TYPE_AVATAR_URLS: + processAvatarURLsMessage(app->_incomingPacket, bytesReceived); break; case PACKET_TYPE_AVATAR_FACE_VIDEO: processAvatarFaceVideoMessage(app->_incomingPacket, bytesReceived); diff --git a/interface/src/Application.h b/interface/src/Application.h index 5464c853f1..e951afc735 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -187,7 +187,7 @@ private: void updateProjectionMatrix(); static bool sendVoxelsOperation(VoxelNode* node, void* extraData); - static void processAvatarVoxelURLMessage(unsigned char* packetData, size_t dataBytes); + static void processAvatarURLsMessage(unsigned char* packetData, size_t dataBytes); static void processAvatarFaceVideoMessage(unsigned char* packetData, size_t dataBytes); static void sendPingPackets(); diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 115ca7aeeb..a88954d64b 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -786,11 +786,12 @@ void Menu::editPreferences() { QUrl avatarVoxelURL(avatarURL->text()); applicationInstance->getAvatar()->getVoxels()->setVoxelURL(avatarVoxelURL); - Avatar::sendAvatarVoxelURLMessage(avatarVoxelURL); QUrl faceModelURL(faceURL->text()); applicationInstance->getAvatar()->getHead().getBlendFace().setModelURL(faceModelURL); + Avatar::sendAvatarURLsMessage(avatarVoxelURL, faceModelURL); + _gyroCameraSensitivity = gyroCameraSensitivity->value(); applicationInstance->getAvatar()->setLeanScale(leanScale->value()); diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index cdfefedaac..a5ed16bcb5 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -61,7 +61,7 @@ const bool usingBigSphereCollisionTest = true; const float chatMessageScale = 0.0015; const float chatMessageHeight = 0.20; -void Avatar::sendAvatarVoxelURLMessage(const QUrl& url) { +void Avatar::sendAvatarURLsMessage(const QUrl& voxelURL, const QUrl& faceURL) { uint16_t ownerID = NodeList::getInstance()->getOwnerID(); if (ownerID == UNKNOWN_NODE_ID) { @@ -71,11 +71,14 @@ void Avatar::sendAvatarVoxelURLMessage(const QUrl& url) { QByteArray message; char packetHeader[MAX_PACKET_HEADER_BYTES]; - int numBytesPacketHeader = populateTypeAndVersion((unsigned char*) packetHeader, PACKET_TYPE_AVATAR_VOXEL_URL); + int numBytesPacketHeader = populateTypeAndVersion((unsigned char*) packetHeader, PACKET_TYPE_AVATAR_URLS); message.append(packetHeader, numBytesPacketHeader); message.append((const char*)&ownerID, sizeof(ownerID)); - message.append(url.toEncoded()); + + QDataStream out(&message, QIODevice::WriteOnly); + out << voxelURL; + out << faceURL; Application::controlledBroadcastToNodes((unsigned char*)message.data(), message.size(), &NODE_TYPE_AVATAR_MIXER, 1); } diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index 3aee9599d2..9108a0f515 100755 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -129,7 +129,7 @@ class Avatar : public AvatarData { Q_OBJECT public: - static void sendAvatarVoxelURLMessage(const QUrl& url); + static void sendAvatarURLsMessage(const QUrl& voxelURL, const QUrl& faceURL); Avatar(Node* owningNode = NULL); ~Avatar(); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 4973579243..dab3d1651e 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -205,12 +205,10 @@ int AvatarData::getBroadcastData(unsigned char* destinationBuffer) { memcpy(destinationBuffer, &_headData->_browAudioLift, sizeof(float)); destinationBuffer += sizeof(float); - /* *destinationBuffer++ = _headData->_blendshapeCoefficients.size(); memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), _headData->_blendshapeCoefficients.size() * sizeof(float)); destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); - */ } // leap hand data @@ -342,12 +340,10 @@ int AvatarData::parseData(unsigned char* sourceBuffer, int numBytes) { memcpy(&_headData->_browAudioLift, sourceBuffer, sizeof(float)); sourceBuffer += sizeof(float); - /* _headData->_blendshapeCoefficients.resize(*sourceBuffer++); memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, _headData->_blendshapeCoefficients.size() * sizeof(float)); sourceBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); - */ } // leap hand data diff --git a/libraries/shared/src/PacketHeaders.cpp b/libraries/shared/src/PacketHeaders.cpp index d31d499814..c3e67fb10f 100644 --- a/libraries/shared/src/PacketHeaders.cpp +++ b/libraries/shared/src/PacketHeaders.cpp @@ -20,8 +20,11 @@ PACKET_VERSION versionForPacketType(PACKET_TYPE type) { return 1; case PACKET_TYPE_HEAD_DATA: - return 6; + return 7; + case PACKET_TYPE_AVATAR_URLS: + return 1; + case PACKET_TYPE_AVATAR_FACE_VIDEO: return 1; diff --git a/libraries/shared/src/PacketHeaders.h b/libraries/shared/src/PacketHeaders.h index 59379d5dd8..6d20eac078 100644 --- a/libraries/shared/src/PacketHeaders.h +++ b/libraries/shared/src/PacketHeaders.h @@ -29,9 +29,8 @@ const PACKET_TYPE PACKET_TYPE_ERASE_VOXEL = 'E'; const PACKET_TYPE PACKET_TYPE_VOXEL_DATA = 'V'; const PACKET_TYPE PACKET_TYPE_VOXEL_DATA_MONOCHROME = 'v'; const PACKET_TYPE PACKET_TYPE_BULK_AVATAR_DATA = 'X'; -const PACKET_TYPE PACKET_TYPE_AVATAR_VOXEL_URL = 'U'; +const PACKET_TYPE PACKET_TYPE_AVATAR_URLS = 'U'; const PACKET_TYPE PACKET_TYPE_AVATAR_FACE_VIDEO = 'F'; -const PACKET_TYPE PACKET_TYPE_AVATAR_FACE_URL = 'f'; const PACKET_TYPE PACKET_TYPE_TRANSMITTER_DATA_V2 = 'T'; const PACKET_TYPE PACKET_TYPE_ENVIRONMENT_DATA = 'e'; const PACKET_TYPE PACKET_TYPE_DOMAIN_LIST_REQUEST = 'L'; From c0496650903bfcde0f8ac613d86613e0a9cc52fc Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 19 Sep 2013 16:42:25 -0700 Subject: [PATCH 14/16] Removed temporary comment. --- interface/src/renderer/FBXReader.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index ce4dc35934..004db26f2f 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -260,16 +260,6 @@ QHash createBlendshapeMap() { FBXGeometry extractFBXGeometry(const FBXNode& node) { FBXGeometry geometry; - // "Objects" - // "Geometry" QVariant(qlonglong, 5386006736) QVariant(QByteArray, "QVariant(QByteArray, "Mesh") - // "Vertices" QVector - // "PolygonVertexIndex" QVector - // "LayerElementNormal" - // "Normals" QVector - // "Geometry" QVariant(qlonglong, 5470612480) QVariant(QByteArray, "BrowsD_LQVariant(QByteArray, "Shape") - // "Vertices" QVector - // "Normals" QVector - foreach (const FBXNode& child, node.children) { if (child.name == "Objects") { foreach (const FBXNode& object, child.children) { From 2f5a5a4b27e49237c30ec0e1777f1ef762ca1067 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 20 Sep 2013 09:52:15 -0700 Subject: [PATCH 15/16] Remove unnecessary, unlucky "13," add note explaining arbitrary translation. --- interface/src/avatar/BlendFace.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 690a278453..0fdd37da2c 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -39,7 +39,7 @@ bool BlendFace::render(float alpha) { glm::quat orientation = _owningHead->getOrientation(); glm::vec3 axis = glm::axis(orientation); glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); - glTranslatef(0.0f, -0.025f, -0.025f); + glTranslatef(0.0f, -0.025f, -0.025f); // temporary fudge factor until we have a better method of per-model positioning const float MODEL_SCALE = 0.0006f; glScalef(_owningHead->getScale() * MODEL_SCALE, _owningHead->getScale() * MODEL_SCALE, -_owningHead->getScale() * MODEL_SCALE); From 77d9e21162e338d5d4c1f0426bc0e21c54699e3e Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 20 Sep 2013 10:06:34 -0700 Subject: [PATCH 16/16] Meant to include this in the last commit. --- interface/src/renderer/FBXReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 004db26f2f..ba898748e2 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -171,7 +171,7 @@ FBXNode parseFBX(QIODevice* device) { // parse the top-level node FBXNode top; - while (device->bytesAvailable() >= 13) { + while (device->bytesAvailable()) { FBXNode next = parseFBXNode(in); if (next.name.isNull()) { return top;