From 9ba3c3719fbbd0ef958a9391282f349ea1f25b0f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 17 Oct 2013 15:45:57 -0700 Subject: [PATCH 01/27] Working on skeleton loading, added support for FBX text format (exported by Blender). --- interface/src/Menu.cpp | 5 + interface/src/avatar/Avatar.cpp | 2 + interface/src/avatar/Avatar.h | 7 +- interface/src/avatar/Body.cpp | 66 +++++++++++ interface/src/avatar/Body.h | 50 ++++++++ interface/src/avatar/Profile.cpp | 12 +- interface/src/avatar/Profile.h | 4 + interface/src/renderer/FBXReader.cpp | 163 +++++++++++++++++++++++---- interface/src/renderer/FBXReader.h | 1 + 9 files changed, 287 insertions(+), 23 deletions(-) create mode 100644 interface/src/avatar/Body.cpp create mode 100644 interface/src/avatar/Body.h diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index f9479865c6..da7ec8f941 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -805,6 +805,11 @@ void Menu::editPreferences() { faceURLEdit->setMinimumWidth(QLINE_MINIMUM_WIDTH); form->addRow("Face URL:", faceURLEdit); + QString skeletonURLString = applicationInstance->getProfile()->getSkeletonModelURL().toString(); + QLineEdit* skeletonURLEdit = new QLineEdit(skeletonURLString); + skeletonURLEdit->setMinimumWidth(QLINE_MINIMUM_WIDTH); + form->addRow("Skeleton URL:", skeletonURLEdit); + QSlider* pupilDilation = new QSlider(Qt::Horizontal); pupilDilation->setValue(applicationInstance->getAvatar()->getHead().getPupilDilation() * pupilDilation->maximum()); form->addRow("Pupil Dilation:", pupilDilation); diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 9c755a24c1..ded8bac53f 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -84,6 +84,7 @@ void Avatar::sendAvatarURLsMessage(const QUrl& voxelURL) { Avatar::Avatar(Node* owningNode) : AvatarData(owningNode), _head(this), + _body(this), _hand(this), _ballSpringsInitialized(false), _bodyYawDelta(0.0f), @@ -420,6 +421,7 @@ void Avatar::simulate(float deltaTime, Transmitter* transmitter) { _head.setPosition(_bodyBall[ BODY_BALL_HEAD_BASE ].position); _head.setSkinColor(glm::vec3(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2])); _head.simulate(deltaTime, false); + _body.simulate(deltaTime); _hand.simulate(deltaTime, false); // use speed and angular velocity to determine walking vs. standing diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index b46f461389..074142cc8f 100755 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -18,6 +18,7 @@ #include "AvatarTouch.h" #include "AvatarVoxelSystem.h" #include "Balls.h" +#include "Body.h" #include "Hand.h" #include "Head.h" #include "InterfaceConfig.h" @@ -150,8 +151,9 @@ public: const glm::vec3& getHeadJointPosition() const { return _skeleton.joint[ AVATAR_JOINT_HEAD_BASE ].position; } float getScale() const { return _scale; } const glm::vec3& getVelocity() const { return _velocity; } - Head& getHead() {return _head; } - Hand& getHand() {return _hand; } + Head& getHead() { return _head; } + Body& getBody() { return _body; } + Hand& getHand() { return _hand; } glm::quat getOrientation() const; glm::quat getWorldAlignedOrientation() const; AvatarVoxelSystem* getVoxels() { return &_voxels; } @@ -196,6 +198,7 @@ protected: }; Head _head; + Body _body; Hand _hand; Skeleton _skeleton; bool _ballSpringsInitialized; diff --git a/interface/src/avatar/Body.cpp b/interface/src/avatar/Body.cpp new file mode 100644 index 0000000000..684c9deb56 --- /dev/null +++ b/interface/src/avatar/Body.cpp @@ -0,0 +1,66 @@ +// +// Body.cpp +// interface +// +// Created by Andrzej Kapolka on 10/17/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include + +#include "Application.h" +#include "Body.h" + +Body::Body(Avatar* owningAvatar) : _owningAvatar(owningAvatar) { + // we may have been created in the network thread, but we live in the main thread + moveToThread(Application::getInstance()->thread()); +} + +void Body::simulate(float deltaTime) { + if (!isActive()) { + return; + } + + // set up joint states on first simulate after load + const FBXGeometry& geometry = _skeletonGeometry->getFBXGeometry(); + if (_jointStates.isEmpty()) { + foreach (const FBXJoint& joint, geometry.joints) { + JointState state; + state.rotation = joint.rotation; + _jointStates.append(state); + } + } + + glm::quat orientation = _owningAvatar->getOrientation(); + const float MODEL_SCALE = 0.0006f; + glm::vec3 scale = glm::vec3(-1.0f, 1.0f, -1.0f) * _owningAvatar->getScale() * MODEL_SCALE; + glm::mat4 baseTransform = glm::translate(_owningAvatar->getPosition()) * glm::mat4_cast(orientation) * glm::scale(scale); + + // update the world space transforms for all joints + for (int i = 0; i < _jointStates.size(); i++) { + JointState& state = _jointStates[i]; + const FBXJoint& joint = geometry.joints.at(i); + if (joint.parentIndex == -1) { + state.transform = baseTransform * geometry.offset * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + + } else { + state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + } + } +} + +bool Body::render(float alpha) { + return false; +} + +void Body::setSkeletonModelURL(const QUrl& url) { + // don't recreate the geometry if it's the same URL + if (_skeletonModelURL == url) { + return; + } + _skeletonModelURL = url; + + _skeletonGeometry = Application::getInstance()->getGeometryCache()->getGeometry(url); +} diff --git a/interface/src/avatar/Body.h b/interface/src/avatar/Body.h new file mode 100644 index 0000000000..5cb6790dbd --- /dev/null +++ b/interface/src/avatar/Body.h @@ -0,0 +1,50 @@ +// +// Body.h +// interface +// +// Created by Andrzej Kapolka on 10/17/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__Body__ +#define __interface__Body__ + +#include +#include + +#include "renderer/GeometryCache.h" + +/// An avatar body with an arbitrary skeleton. +class Body : public QObject { + Q_OBJECT + +public: + + Body(Avatar* owningAvatar); + + bool isActive() const { return _skeletonGeometry && _skeletonGeometry->isLoaded(); } + + void simulate(float deltaTime); + bool render(float alpha); + + Q_INVOKABLE void setSkeletonModelURL(const QUrl& url); + const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; } + +private: + + Avatar* _owningAvatar; + + QUrl _skeletonModelURL; + + QSharedPointer _skeletonGeometry; + + class JointState { + public: + glm::quat rotation; + glm::mat4 transform; + }; + + QVector _jointStates; +}; + +#endif /* defined(__interface__Body__) */ diff --git a/interface/src/avatar/Profile.cpp b/interface/src/avatar/Profile.cpp index 4b89849620..418b842e40 100644 --- a/interface/src/avatar/Profile.cpp +++ b/interface/src/avatar/Profile.cpp @@ -49,6 +49,14 @@ void Profile::setFaceModelURL(const QUrl& faceModelURL) { Q_ARG(QUrl, _faceModelURL)); } +void Profile::setSkeletonModelURL(const QUrl& skeletonModelURL) { + _skeletonModelURL = skeletonModelURL; + + QMetaObject::invokeMethod(&Application::getInstance()->getAvatar()->getBody(), + "setSkeletonModelURL", + Q_ARG(QUrl, _skeletonModelURL)); +} + void Profile::updateDomain(const QString& domain) { if (_lastDomain != domain) { _lastDomain = domain; @@ -91,6 +99,7 @@ void Profile::saveData(QSettings* settings) { settings->setValue("username", _username); settings->setValue("UUID", _uuid); settings->setValue("faceModelURL", _faceModelURL); + settings->setValue("skeletonModelURL", _skeletonModelURL); settings->endGroup(); } @@ -101,6 +110,7 @@ void Profile::loadData(QSettings* settings) { _username = settings->value("username").toString(); this->setUUID(settings->value("UUID").toUuid()); _faceModelURL = settings->value("faceModelURL").toUrl(); + _skeletonModelURL = settings->value("skeletonModelURL").toUrl(); settings->endGroup(); -} \ No newline at end of file +} diff --git a/interface/src/avatar/Profile.h b/interface/src/avatar/Profile.h index 46086d1476..eb3f531819 100644 --- a/interface/src/avatar/Profile.h +++ b/interface/src/avatar/Profile.h @@ -29,6 +29,9 @@ public: void setFaceModelURL(const QUrl& faceModelURL); const QUrl& getFaceModelURL() const { return _faceModelURL; } + void setSkeletonModelURL(const QUrl& skeletonModelURL); + const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; } + void updateDomain(const QString& domain); void updatePosition(const glm::vec3 position); @@ -43,6 +46,7 @@ private: QString _lastDomain; glm::vec3 _lastPosition; QUrl _faceModelURL; + QUrl _skeletonModelURL; }; #endif /* defined(__hifi__Profile__) */ diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index b2d193d106..3d6d7ae3a6 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -19,7 +20,7 @@ using namespace std; -template QVariant readArray(QDataStream& in) { +template QVariant readBinaryArray(QDataStream& in) { quint32 arrayLength; quint32 encoding; quint32 compressedLength; @@ -54,7 +55,7 @@ template QVariant readArray(QDataStream& in) { return QVariant::fromValue(values); } -QVariant parseFBXProperty(QDataStream& in) { +QVariant parseBinaryFBXProperty(QDataStream& in) { char ch; in.device()->getChar(&ch); switch (ch) { @@ -89,19 +90,19 @@ QVariant parseFBXProperty(QDataStream& in) { return QVariant::fromValue(value); } case 'f': { - return readArray(in); + return readBinaryArray(in); } case 'd': { - return readArray(in); + return readBinaryArray(in); } case 'l': { - return readArray(in); + return readBinaryArray(in); } case 'i': { - return readArray(in); + return readBinaryArray(in); } case 'b': { - return readArray(in); + return readBinaryArray(in); } case 'S': case 'R': { @@ -114,7 +115,7 @@ QVariant parseFBXProperty(QDataStream& in) { } } -FBXNode parseFBXNode(QDataStream& in) { +FBXNode parseBinaryFBXNode(QDataStream& in) { quint32 endOffset; quint32 propertyCount; quint32 propertyListLength; @@ -134,11 +135,11 @@ FBXNode parseFBXNode(QDataStream& in) { node.name = in.device()->read(nameLength); for (int i = 0; i < propertyCount; i++) { - node.properties.append(parseFBXProperty(in)); + node.properties.append(parseBinaryFBXProperty(in)); } while (endOffset > in.device()->pos()) { - FBXNode child = parseFBXNode(in); + FBXNode child = parseBinaryFBXNode(in); if (child.name.isNull()) { return node; @@ -150,28 +151,149 @@ FBXNode parseFBXNode(QDataStream& in) { return node; } +class Tokenizer { +public: + + Tokenizer(QIODevice* device) : _device(device), _pushedBackToken(-1) { } + + enum SpecialToken { DATUM_TOKEN = 0x100 }; + + int nextToken(); + const QByteArray& getDatum() const { return _datum; } + + void pushBackToken(int token) { _pushedBackToken = token; } + +private: + + QIODevice* _device; + QByteArray _datum; + int _pushedBackToken; +}; + +int Tokenizer::nextToken() { + if (_pushedBackToken != -1) { + int token = _pushedBackToken; + _pushedBackToken = -1; + return token; + } + + char ch; + while (_device->getChar(&ch)) { + if (QChar(ch).isSpace()) { + continue; // skip whitespace + } + switch (ch) { + case ';': + _device->readLine(); // skip the comment + break; + + case ':': + case '{': + case '}': + case ',': + return ch; // special punctuation + + case '\"': + _datum = ""; + while (_device->getChar(&ch)) { + if (ch == '\"') { // end on closing quote + break; + } + if (ch == '\\') { // handle escaped quotes + if (_device->getChar(&ch) && ch != '\"') { + _datum.append('\\'); + } + } + _datum.append(ch); + } + return DATUM_TOKEN; + + default: + _datum = ""; + _datum.append(ch); + while (_device->getChar(&ch)) { + if (QChar(ch).isSpace() || ch == ';' || ch == ':' || ch == '{' || ch == '}' || ch == ',' || ch == '\"') { + _device->ungetChar(ch); // read until we encounter a special character, then replace it + break; + } + _datum.append(ch); + } + return DATUM_TOKEN; + } + } + return -1; +} + +FBXNode parseTextFBXNode(Tokenizer& tokenizer) { + FBXNode node; + + if (tokenizer.nextToken() != Tokenizer::DATUM_TOKEN) { + return node; + } + node.name = tokenizer.getDatum(); + + if (tokenizer.nextToken() != ':') { + return node; + } + + int token; + bool expectingDatum = true; + while ((token = tokenizer.nextToken()) != -1) { + if (token == '{') { + for (FBXNode child = parseTextFBXNode(tokenizer); !child.name.isNull(); child = parseTextFBXNode(tokenizer)) { + node.children.append(child); + } + return node; + } + if (token == ',') { + expectingDatum = true; + + } else if (token == Tokenizer::DATUM_TOKEN && expectingDatum) { + node.properties.append(tokenizer.getDatum()); + expectingDatum = false; + + } else { + tokenizer.pushBackToken(token); + return node; + } + } + + return node; +} + FBXNode parseFBX(QIODevice* device) { + // verify the prolog + const QByteArray BINARY_PROLOG = "Kaydara FBX Binary "; + if (device->peek(BINARY_PROLOG.size()) != BINARY_PROLOG) { + // parse as a text file + FBXNode top; + Tokenizer tokenizer(device); + while (device->bytesAvailable()) { + FBXNode next = parseTextFBXNode(tokenizer); + if (next.name.isNull()) { + return top; + + } else { + top.children.append(next); + } + } + return top; + } QDataStream in(device); in.setByteOrder(QDataStream::LittleEndian); in.setVersion(QDataStream::Qt_4_5); // for single/double precision switch // see http://code.blender.org/index.php/2013/08/fbx-binary-file-format-specification/ for an explanation - // of the FBX format - - // verify the prolog - const QByteArray EXPECTED_PROLOG = "Kaydara FBX Binary "; - if (device->read(EXPECTED_PROLOG.size()) != EXPECTED_PROLOG) { - throw QString("Invalid header."); - } + // of the FBX binary format // skip the rest of the header const int HEADER_SIZE = 27; - in.skipRawData(HEADER_SIZE - EXPECTED_PROLOG.size()); + in.skipRawData(HEADER_SIZE); // parse the top-level node FBXNode top; while (device->bytesAvailable()) { - FBXNode next = parseFBXNode(in); + FBXNode next = parseBinaryFBXNode(in); if (next.name.isNull()) { return top; @@ -750,7 +872,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) joint.transform = geometry.joints.at(joint.parentIndex).transform * model.preRotation * glm::mat4_cast(model.rotation) * model.postRotation; } - geometry.joints.append(joint); + geometry.joints.append(joint); + geometry.jointIndices.insert(model.name, geometry.joints.size() - 1); } // find our special joints diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 85e463f8bf..e3b29bd1fe 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -99,6 +99,7 @@ class FBXGeometry { public: QVector joints; + QHash jointIndices; QVector meshes; From c8b2ed0213bef74565e7e4848e58fd58c4d4e3c6 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 17 Oct 2013 17:30:34 -0700 Subject: [PATCH 02/27] More progress on reading skeletons. --- interface/src/Application.cpp | 4 +- interface/src/DataServerClient.cpp | 20 ++++ interface/src/DataServerClient.h | 1 + interface/src/Menu.cpp | 11 +++ interface/src/avatar/Avatar.cpp | 6 +- interface/src/avatar/Body.cpp | 7 +- interface/src/avatar/MyAvatar.cpp | 7 +- interface/src/renderer/FBXReader.cpp | 132 ++++++++++++++----------- interface/src/renderer/GeometryCache.h | 2 +- 9 files changed, 124 insertions(+), 66 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f9424500a7..8113057210 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1321,7 +1321,8 @@ void Application::processAvatarURLsMessage(unsigned char* packetData, size_t dat QMetaObject::invokeMethod(avatar->getVoxels(), "setVoxelURL", Q_ARG(QUrl, voxelURL)); // use this timing to as the data-server for an updated mesh for this avatar (if we have UUID) - DataServerClient::getValueForKeyAndUUID(DataServerKey::FaceMeshURL, avatar->getUUID()); + DataServerClient::getValuesForKeysAndUUID(QStringList() << DataServerKey::FaceMeshURL << DataServerKey::SkeletonURL, + avatar->getUUID()); } void Application::processAvatarFaceVideoMessage(unsigned char* packetData, size_t dataBytes) { @@ -1652,6 +1653,7 @@ void Application::init() { if (!_profile.getUsername().isEmpty()) { // we have a username for this avatar, ask the data-server for the mesh URL for this avatar DataServerClient::getClientValueForKey(DataServerKey::FaceMeshURL); + DataServerClient::getClientValueForKey(DataServerKey::SkeletonURL); } // Set up VoxelSystem after loading preferences so we can get the desired max voxel count diff --git a/interface/src/DataServerClient.cpp b/interface/src/DataServerClient.cpp index 26e297375c..ef5293fa54 100644 --- a/interface/src/DataServerClient.cpp +++ b/interface/src/DataServerClient.cpp @@ -148,6 +148,26 @@ void DataServerClient::processSendFromDataServer(unsigned char* packetData, int } } } + } else if (keyList[i] == DataServerKey::SkeletonURL) { + + if (userUUID.isNull() || userUUID == Application::getInstance()->getProfile()->getUUID()) { + qDebug("Changing user's skeleton URL to %s\n", valueList[0].toLocal8Bit().constData()); + Application::getInstance()->getProfile()->setSkeletonModelURL(QUrl(valueList[0])); + } else { + // skeleton URL for a UUID, find avatar in our list + NodeList* nodeList = NodeList::getInstance(); + for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { + if (node->getLinkedData() != NULL && node->getType() == NODE_TYPE_AGENT) { + Avatar* avatar = (Avatar *) node->getLinkedData(); + + if (avatar->getUUID() == userUUID) { + QMetaObject::invokeMethod(&avatar->getBody(), + "setSkeletonModelURL", + Q_ARG(QUrl, QUrl(valueList[0]))); + } + } + } + } } else if (keyList[i] == DataServerKey::Domain && keyList[i + 1] == DataServerKey::Position && valueList[i] != " " && valueList[i + 1] != " ") { diff --git a/interface/src/DataServerClient.h b/interface/src/DataServerClient.h index dabb1c822c..2331cae1e9 100644 --- a/interface/src/DataServerClient.h +++ b/interface/src/DataServerClient.h @@ -38,6 +38,7 @@ private: namespace DataServerKey { const QString Domain = "domain"; const QString FaceMeshURL = "mesh"; + const QString SkeletonURL = "skeleton"; const QString Position = "position"; const QString UUID = "uuid"; } diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index da7ec8f941..013f1d461b 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -862,6 +862,17 @@ void Menu::editPreferences() { faceModelURL.toString().toLocal8Bit().constData()); } + QUrl skeletonModelURL(skeletonURLEdit->text()); + + if (skeletonModelURL.toString() != skeletonURLString) { + // change the skeletonModelURL in the profile, it will also update this user's Body + applicationInstance->getProfile()->setSkeletonModelURL(skeletonModelURL); + + // send the new skeleton model URL to the data-server (if we have a client UUID) + DataServerClient::putValueForKey(DataServerKey::SkeletonURL, + skeletonModelURL.toString().toLocal8Bit().constData()); + } + QUrl avatarVoxelURL(avatarURL->text()); applicationInstance->getAvatar()->getVoxels()->setVoxelURL(avatarVoxelURL); diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index ded8bac53f..f027df2c25 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -756,7 +756,7 @@ void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { if (alpha > 0.0f) { _head.getFace().render(1.0f); } - } else if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) { + } else if (renderAvatarBalls || !(_voxels.getVoxelURL().isValid() || _body.isActive())) { // Render the body as balls and cones glm::vec3 skinColor(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2]); glm::vec3 darkSkinColor(DARK_SKIN_COLOR[0], DARK_SKIN_COLOR[1], DARK_SKIN_COLOR[2]); @@ -821,7 +821,9 @@ void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { // Render the body's voxels and head float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); if (alpha > 0.0f) { - _voxels.render(false); + if (!_body.render(alpha)) { + _voxels.render(false); + } _head.render(alpha, false); } } diff --git a/interface/src/avatar/Body.cpp b/interface/src/avatar/Body.cpp index 684c9deb56..20f30d7b3f 100644 --- a/interface/src/avatar/Body.cpp +++ b/interface/src/avatar/Body.cpp @@ -52,7 +52,11 @@ void Body::simulate(float deltaTime) { } bool Body::render(float alpha) { - return false; + if (_jointStates.isEmpty()) { + return false; + } + + return true; } void Body::setSkeletonModelURL(const QUrl& url) { @@ -61,6 +65,5 @@ void Body::setSkeletonModelURL(const QUrl& url) { return; } _skeletonModelURL = url; - _skeletonGeometry = Application::getInstance()->getGeometryCache()->getGeometry(url); } diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index e1c18089fd..01a6c356fc 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -322,6 +322,7 @@ void MyAvatar::simulate(float deltaTime, Transmitter* transmitter) { _head.setScale(_scale); _head.setSkinColor(glm::vec3(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2])); _head.simulate(deltaTime, true); + _body.simulate(deltaTime); _hand.simulate(deltaTime, true); const float WALKING_SPEED_THRESHOLD = 0.2f; @@ -610,7 +611,7 @@ void MyAvatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { if (alpha > 0.0f) { _head.getFace().render(1.0f); } - } else if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) { + } else if (renderAvatarBalls || !(_voxels.getVoxelURL().isValid() || _body.isActive())) { // Render the body as balls and cones glm::vec3 skinColor(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2]); glm::vec3 darkSkinColor(DARK_SKIN_COLOR[0], DARK_SKIN_COLOR[1], DARK_SKIN_COLOR[2]); @@ -685,7 +686,9 @@ void MyAvatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { // Render the body's voxels and head float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); if (alpha > 0.0f) { - _voxels.render(false); + if (!_body.render(alpha)) { + _voxels.render(false); + } _head.render(alpha, true); } } diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 3d6d7ae3a6..8e4b95f5c3 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -431,15 +431,16 @@ public: int parentIndex; }; -glm::mat4 getGlobalTransform(const QMultiHash& parentMap, const QHash& models, qint64 nodeID) { +glm::mat4 getGlobalTransform(const QMultiHash& parentMap, + const QHash& models, QString nodeID) { glm::mat4 globalTransform; - while (nodeID != 0) { + while (!nodeID.isNull()) { const Model& model = models.value(nodeID); globalTransform = model.preRotation * glm::mat4_cast(model.rotation) * model.postRotation * globalTransform; - QList parentIDs = parentMap.values(nodeID); - nodeID = 0; - foreach (qint64 parentID, parentIDs) { + QList parentIDs = parentMap.values(nodeID); + nodeID = QString(); + foreach (const QString& parentID, parentIDs) { if (models.contains(parentID)) { nodeID = parentID; break; @@ -452,7 +453,7 @@ glm::mat4 getGlobalTransform(const QMultiHash& parentMap, const class ExtractedBlendshape { public: - qint64 id; + QString id; FBXBlendshape blendshape; }; @@ -482,13 +483,13 @@ public: glm::mat4 transformLink; }; -void appendModelIDs(qint64 parentID, const QMultiHash& childMap, - QHash& models, QVector& modelIDs) { - if (parentID != 0) { +void appendModelIDs(const QString& parentID, const QMultiHash& childMap, + QHash& models, QVector& modelIDs) { + if (models.contains(parentID)) { modelIDs.append(parentID); } int parentIndex = modelIDs.size() - 1; - foreach (qint64 childID, childMap.values(parentID)) { + foreach (const QString& childID, childMap.values(parentID)) { if (models.contains(childID)) { models[childID].parentIndex = parentIndex; appendModelIDs(childID, childMap, models, modelIDs); @@ -497,24 +498,24 @@ void appendModelIDs(qint64 parentID, const QMultiHash& childMap, } FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) { - QHash meshes; + QHash meshes; QVector blendshapes; - QMultiHash parentMap; - QMultiHash childMap; - QHash models; - QHash clusters; - QHash textureFilenames; - QHash materials; - QHash diffuseTextures; - QHash bumpTextures; + QMultiHash parentMap; + QMultiHash childMap; + QHash models; + QHash clusters; + QHash textureFilenames; + QHash materials; + QHash diffuseTextures; + QHash bumpTextures; QVariantHash joints = mapping.value("joint").toHash(); QByteArray jointEyeLeftName = joints.value("jointEyeLeft", "jointEyeLeft").toByteArray(); QByteArray jointEyeRightName = joints.value("jointEyeRight", "jointEyeRight").toByteArray(); QByteArray jointNeckName = joints.value("jointNeck", "jointNeck").toByteArray(); - qint64 jointEyeLeftID = 0; - qint64 jointEyeRightID = 0; - qint64 jointNeckID = 0; + QString jointEyeLeftID; + QString jointEyeRightID; + QString jointNeckID; QVariantHash blendshapeMappings = mapping.value("bs").toHash(); QHash > blendshapeIndices; @@ -534,7 +535,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } } } - QHash > blendshapeChannelIndices; + QHash > blendshapeChannelIndices; foreach (const FBXNode& child, node.children) { if (child.name == "Objects") { @@ -653,10 +654,10 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) beginIndex = endIndex; } } - meshes.insert(object.properties.at(0).value(), mesh); + meshes.insert(object.properties.at(0).toString(), mesh); } else { // object.properties.at(2) == "Shape" - ExtractedBlendshape extracted = { object.properties.at(0).value() }; + ExtractedBlendshape extracted = { object.properties.at(0).toString() }; foreach (const FBXNode& data, object.children) { if (data.name == "Indexes") { @@ -678,13 +679,13 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QByteArray name = object.properties.at(1).toByteArray(); name = name.left(name.indexOf('\0')); if (name == jointEyeLeftName || name == "EyeL" || name == "joint_Leye") { - jointEyeLeftID = object.properties.at(0).value(); + jointEyeLeftID = object.properties.at(0).toString(); } else if (name == jointEyeRightName || name == "EyeR" || name == "joint_Reye") { - jointEyeRightID = object.properties.at(0).value(); + jointEyeRightID = object.properties.at(0).toString(); } else if (name == jointNeckName || name == "NeckRot" || name == "joint_neck") { - jointNeckID = object.properties.at(0).value(); + jointNeckID = object.properties.at(0).toString(); } glm::vec3 translation; glm::vec3 preRotation, rotation, postRotation; @@ -740,7 +741,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) model.rotation = glm::quat(glm::radians(rotation)); model.postRotation = glm::mat4_cast(glm::quat(glm::radians(postRotation))) * glm::translate(-rotationPivot) * glm::translate(scalePivot) * glm::scale(scale) * glm::translate(-scalePivot); - models.insert(object.properties.at(0).value(), model); + models.insert(object.properties.at(0).toString(), model); } else if (object.name == "Texture") { foreach (const FBXNode& subobject, object.children) { @@ -748,7 +749,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) // trim off any path information QByteArray filename = subobject.properties.at(0).toByteArray(); filename = filename.mid(qMax(filename.lastIndexOf('\\'), filename.lastIndexOf('/')) + 1); - textureFilenames.insert(object.properties.at(0).value(), filename); + textureFilenames.insert(object.properties.at(0).toString(), filename); } } } else if (object.name == "Material") { @@ -774,7 +775,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } } } - materials.insert(object.properties.at(0).value(), material); + materials.insert(object.properties.at(0).toString(), material); } else if (object.name == "Deformer") { if (object.properties.at(2) == "Cluster") { @@ -791,7 +792,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) cluster.transformLink = createMat4(values); } } - clusters.insert(object.properties.at(0).value(), cluster); + clusters.insert(object.properties.at(0).toString(), cluster); } else if (object.properties.at(2) == "BlendShapeChannel") { QByteArray name = object.properties.at(1).toByteArray(); @@ -800,26 +801,26 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) // try everything after the dot name = name.mid(name.lastIndexOf('.') + 1); } - blendshapeChannelIndices.insert(object.properties.at(0).value(), + blendshapeChannelIndices.insert(object.properties.at(0).toString(), blendshapeIndices.value(name)); } } } } else if (child.name == "Connections") { foreach (const FBXNode& connection, child.children) { - if (connection.name == "C") { + if (connection.name == "C" || connection.name == "Connect") { if (connection.properties.at(0) == "OP") { if (connection.properties.at(3) == "DiffuseColor") { - diffuseTextures.insert(connection.properties.at(2).value(), - connection.properties.at(1).value()); + diffuseTextures.insert(connection.properties.at(2).toString(), + connection.properties.at(1).toString()); } else if (connection.properties.at(3) == "Bump") { - bumpTextures.insert(connection.properties.at(2).value(), - connection.properties.at(1).value()); + bumpTextures.insert(connection.properties.at(2).toString(), + connection.properties.at(1).toString()); } } - parentMap.insert(connection.properties.at(1).value(), connection.properties.at(2).value()); - childMap.insert(connection.properties.at(2).value(), connection.properties.at(1).value()); + parentMap.insert(connection.properties.at(1).toString(), connection.properties.at(2).toString()); + childMap.insert(connection.properties.at(2).toString(), connection.properties.at(1).toString()); } } } @@ -827,10 +828,10 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) // assign the blendshapes to their corresponding meshes foreach (const ExtractedBlendshape& extracted, blendshapes) { - qint64 blendshapeChannelID = parentMap.value(extracted.id); + QString blendshapeChannelID = parentMap.value(extracted.id); QPair index = blendshapeChannelIndices.value(blendshapeChannelID); - qint64 blendshapeID = parentMap.value(blendshapeChannelID); - qint64 meshID = parentMap.value(blendshapeID); + QString blendshapeID = parentMap.value(blendshapeChannelID); + QString meshID = parentMap.value(blendshapeID); FBXMesh& mesh = meshes[meshID]; mesh.blendshapes.resize(max(mesh.blendshapes.size(), index.first + 1)); mesh.blendshapes[index.first] = extracted.blendshape; @@ -854,11 +855,26 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::scale(offsetScale, offsetScale, offsetScale); // get the list of models in depth-first traversal order - QVector modelIDs; - appendModelIDs(0, childMap, models, modelIDs); + QVector modelIDs; + if (!models.isEmpty()) { + QString top = models.constBegin().key(); + forever { + foreach (const QString& name, parentMap.values(top)) { + if (models.contains(name)) { + top = name; + goto outerContinue; + } + } + top = parentMap.value(top); + break; + + outerContinue: ; + } + appendModelIDs(top, childMap, models, modelIDs); + } // convert the models to joints - foreach (qint64 modelID, modelIDs) { + foreach (const QString& modelID, modelIDs) { const Model& model = models[modelID]; FBXJoint joint; joint.parentIndex = model.parentIndex; @@ -889,17 +905,17 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QVariantHash springs = mapping.value("spring").toHash(); QVariant defaultSpring = springs.value("default"); - for (QHash::iterator it = meshes.begin(); it != meshes.end(); it++) { + for (QHash::iterator it = meshes.begin(); it != meshes.end(); it++) { FBXMesh& mesh = it.value(); // accumulate local transforms - qint64 modelID = parentMap.value(it.key()); + QString modelID = parentMap.value(it.key()); mesh.springiness = springs.value(models.value(modelID).name, defaultSpring).toFloat(); glm::mat4 modelTransform = getGlobalTransform(parentMap, models, modelID); // look for textures, material properties int partIndex = 0; - foreach (qint64 childID, childMap.values(modelID)) { + foreach (const QString& childID, childMap.values(modelID)) { if (!materials.contains(childID) || partIndex >= mesh.parts.size()) { continue; } @@ -908,21 +924,21 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) part.diffuseColor = material.diffuse; part.specularColor = material.specular; part.shininess = material.shininess; - qint64 diffuseTextureID = diffuseTextures.value(childID); - if (diffuseTextureID != 0) { + QString diffuseTextureID = diffuseTextures.value(childID); + if (!diffuseTextureID.isNull()) { part.diffuseFilename = textureFilenames.value(diffuseTextureID); } - qint64 bumpTextureID = bumpTextures.value(childID); - if (bumpTextureID != 0) { + QString bumpTextureID = bumpTextures.value(childID); + if (!bumpTextureID.isNull()) { part.normalFilename = textureFilenames.value(bumpTextureID); } } // find the clusters with which the mesh is associated mesh.isEye = false; - QVector clusterIDs; - foreach (qint64 childID, childMap.values(it.key())) { - foreach (qint64 clusterID, childMap.values(childID)) { + QVector clusterIDs; + foreach (const QString& childID, childMap.values(it.key())) { + foreach (const QString& clusterID, childMap.values(childID)) { if (!clusters.contains(clusterID)) { continue; } @@ -930,7 +946,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) const Cluster& cluster = clusters[clusterID]; clusterIDs.append(clusterID); - qint64 jointID = childMap.value(clusterID); + QString jointID = childMap.value(clusterID); if (jointID == jointEyeLeftID || jointID == jointEyeRightID) { mesh.isEye = true; } @@ -954,7 +970,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) mesh.clusterIndices.resize(mesh.vertices.size()); mesh.clusterWeights.resize(mesh.vertices.size()); for (int i = 0; i < clusterIDs.size(); i++) { - qint64 clusterID = clusterIDs.at(i); + QString clusterID = clusterIDs.at(i); const Cluster& cluster = clusters[clusterID]; for (int j = 0; j < cluster.indices.size(); j++) { diff --git a/interface/src/renderer/GeometryCache.h b/interface/src/renderer/GeometryCache.h index ac6669ad2f..d880d25855 100644 --- a/interface/src/renderer/GeometryCache.h +++ b/interface/src/renderer/GeometryCache.h @@ -57,7 +57,7 @@ public: NetworkGeometry(const QUrl& url); ~NetworkGeometry(); - bool isLoaded() const { return !_meshes.isEmpty(); } + bool isLoaded() const { return !_geometry.joints.isEmpty(); } const FBXGeometry& getFBXGeometry() const { return _geometry; } const QVector& getMeshes() const { return _meshes; } From cd902c5e853e5c7ce9b0934ebc9d1885e4f1e039 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 14:09:12 -0700 Subject: [PATCH 03/27] More work on importing the version of FBX exported by Blender. --- interface/src/avatar/Body.cpp | 14 ++++++++- interface/src/renderer/FBXReader.cpp | 44 ++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/interface/src/avatar/Body.cpp b/interface/src/avatar/Body.cpp index 20f30d7b3f..857edeff13 100644 --- a/interface/src/avatar/Body.cpp +++ b/interface/src/avatar/Body.cpp @@ -32,7 +32,7 @@ void Body::simulate(float deltaTime) { } glm::quat orientation = _owningAvatar->getOrientation(); - const float MODEL_SCALE = 0.0006f; + const float MODEL_SCALE = 0.05f; glm::vec3 scale = glm::vec3(-1.0f, 1.0f, -1.0f) * _owningAvatar->getScale() * MODEL_SCALE; glm::mat4 baseTransform = glm::translate(_owningAvatar->getPosition()) * glm::mat4_cast(orientation) * glm::scale(scale); @@ -56,6 +56,18 @@ bool Body::render(float alpha) { return false; } + glColor4f(1.0f, 1.0f, 1.0f, alpha); + + for (int i = 0; i < _jointStates.size(); i++) { + const JointState& state = _jointStates[i]; + glPushMatrix(); + glMultMatrixf((const GLfloat*)&state.transform); + + glutSolidSphere(0.2f, 10, 10); + + glPopMatrix(); + } + return true; } diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 8e4b95f5c3..9b6cf9dd78 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -693,7 +693,47 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::vec3 scalePivot, rotationPivot; Model model = { name }; foreach (const FBXNode& subobject, object.children) { - if (subobject.name == "Properties70") { + if (subobject.name == "Properties60") { + foreach (const FBXNode& property, subobject.children) { + if (property.name == "Property") { + if (property.properties.at(0) == "Lcl Translation") { + translation = glm::vec3(property.properties.at(3).value(), + property.properties.at(4).value(), + property.properties.at(5).value()); + + } else if (property.properties.at(0) == "RotationPivot") { + rotationPivot = glm::vec3(property.properties.at(3).value(), + property.properties.at(4).value(), + property.properties.at(5).value()); + + } else if (property.properties.at(0) == "PreRotation") { + preRotation = glm::vec3(property.properties.at(3).value(), + property.properties.at(4).value(), + property.properties.at(5).value()); + + } else if (property.properties.at(0) == "Lcl Rotation") { + rotation = glm::vec3(property.properties.at(3).value(), + property.properties.at(4).value(), + property.properties.at(5).value()); + + } else if (property.properties.at(0) == "PostRotation") { + postRotation = glm::vec3(property.properties.at(3).value(), + property.properties.at(4).value(), + property.properties.at(5).value()); + + } else if (property.properties.at(0) == "ScalingPivot") { + scalePivot = glm::vec3(property.properties.at(3).value(), + property.properties.at(4).value(), + property.properties.at(5).value()); + + } else if (property.properties.at(0) == "Lcl Scaling") { + scale = glm::vec3(property.properties.at(3).value(), + property.properties.at(4).value(), + property.properties.at(5).value()); + } + } + } + } else if (subobject.name == "Properties70") { foreach (const FBXNode& property, subobject.children) { if (property.name == "P") { if (property.properties.at(0) == "Lcl Translation") { @@ -737,7 +777,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } // see FBX documentation, http://download.autodesk.com/us/fbx/20112/FBX_SDK_HELP/index.html model.preRotation = glm::translate(translation) * glm::translate(rotationPivot) * - glm::mat4_cast(glm::quat(glm::radians(preRotation))); + glm::mat4_cast(glm::quat(glm::radians(preRotation))); model.rotation = glm::quat(glm::radians(rotation)); model.postRotation = glm::mat4_cast(glm::quat(glm::radians(postRotation))) * glm::translate(-rotationPivot) * glm::translate(scalePivot) * glm::scale(scale) * glm::translate(-scalePivot); From b84a5679ddb48859f1276ceaf98f39f9813d7201 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 14:31:56 -0700 Subject: [PATCH 04/27] Fixed merge error. --- interface/src/renderer/FBXReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index f65ceac057..7c4820bfbc 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -793,7 +793,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) model.postRotation = glm::mat4_cast(glm::quat(glm::radians(postRotation))) * glm::translate(-rotationPivot) * glm::translate(scalePivot) * glm::scale(scale) * glm::translate(-scalePivot); - models.insert(object.properties.at(0).value(), model); + models.insert(object.properties.at(0).toString(), model); } else if (object.name == "Texture") { foreach (const FBXNode& subobject, object.children) { From 7e174093e4adaf3484f30b261f3463fde2e472d4 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 17:26:03 -0700 Subject: [PATCH 05/27] Factoring the FBX model code out to a shared base class. --- .../shaders/{blendface.frag => model.frag} | 0 .../shaders/{blendface.vert => model.vert} | 0 .../{skin_blendface.vert => skin_model.vert} | 0 interface/src/Application.cpp | 6 +- interface/src/DataServerClient.cpp | 10 +- interface/src/Menu.cpp | 2 +- interface/src/avatar/Avatar.cpp | 19 +- interface/src/avatar/Avatar.h | 6 +- interface/src/avatar/BlendFace.cpp | 430 ------------------ interface/src/avatar/BlendFace.h | 92 ---- interface/src/avatar/Body.cpp | 81 ---- interface/src/avatar/Body.h | 50 -- interface/src/avatar/FaceModel.cpp | 55 +++ interface/src/avatar/FaceModel.h | 39 ++ interface/src/avatar/Head.cpp | 16 +- interface/src/avatar/Head.h | 14 +- interface/src/avatar/MyAvatar.cpp | 22 +- interface/src/avatar/Profile.cpp | 10 +- interface/src/avatar/SkeletonModel.cpp | 28 ++ interface/src/avatar/SkeletonModel.h | 31 ++ .../src/avatar/{Face.cpp => VideoFace.cpp} | 38 +- interface/src/avatar/{Face.h => VideoFace.h} | 14 +- interface/src/devices/Webcam.cpp | 2 +- interface/src/renderer/FBXReader.cpp | 14 +- interface/src/renderer/FBXReader.h | 1 + 25 files changed, 239 insertions(+), 741 deletions(-) rename interface/resources/shaders/{blendface.frag => model.frag} (100%) rename interface/resources/shaders/{blendface.vert => model.vert} (100%) rename interface/resources/shaders/{skin_blendface.vert => skin_model.vert} (100%) delete mode 100644 interface/src/avatar/BlendFace.cpp delete mode 100644 interface/src/avatar/BlendFace.h delete mode 100644 interface/src/avatar/Body.cpp delete mode 100644 interface/src/avatar/Body.h create mode 100644 interface/src/avatar/FaceModel.cpp create mode 100644 interface/src/avatar/FaceModel.h create mode 100644 interface/src/avatar/SkeletonModel.cpp create mode 100644 interface/src/avatar/SkeletonModel.h rename interface/src/avatar/{Face.cpp => VideoFace.cpp} (95%) rename interface/src/avatar/{Face.h => VideoFace.h} (89%) diff --git a/interface/resources/shaders/blendface.frag b/interface/resources/shaders/model.frag similarity index 100% rename from interface/resources/shaders/blendface.frag rename to interface/resources/shaders/model.frag diff --git a/interface/resources/shaders/blendface.vert b/interface/resources/shaders/model.vert similarity index 100% rename from interface/resources/shaders/blendface.vert rename to interface/resources/shaders/model.vert diff --git a/interface/resources/shaders/skin_blendface.vert b/interface/resources/shaders/skin_model.vert similarity index 100% rename from interface/resources/shaders/skin_blendface.vert rename to interface/resources/shaders/skin_model.vert diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index dd69880eea..e7e2f4699b 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -363,10 +363,10 @@ void Application::paintGL() { if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { _myCamera.setTightness (100.0f); glm::vec3 targetPosition = _myAvatar.getUprightHeadPosition(); - if (_myAvatar.getHead().getBlendFace().isActive()) { + if (_myAvatar.getHead().getFaceModel().isActive()) { // make sure we're aligned to the blend face eyes glm::vec3 leftEyePosition, rightEyePosition; - if (_myAvatar.getHead().getBlendFace().getEyePositions(leftEyePosition, rightEyePosition, true)) { + if (_myAvatar.getHead().getFaceModel().getEyePositions(leftEyePosition, rightEyePosition)) { targetPosition = (leftEyePosition + rightEyePosition) * 0.5f; } } @@ -1332,7 +1332,7 @@ void Application::processAvatarFaceVideoMessage(unsigned char* packetData, size_ if (!avatar) { return; } - avatar->getHead().getFace().processVideoMessage(packetData, dataBytes); + avatar->getHead().getVideoFace().processVideoMessage(packetData, dataBytes); } void Application::checkBandwidthMeterClick() { diff --git a/interface/src/DataServerClient.cpp b/interface/src/DataServerClient.cpp index ef5293fa54..6fe864d13a 100644 --- a/interface/src/DataServerClient.cpp +++ b/interface/src/DataServerClient.cpp @@ -141,9 +141,8 @@ void DataServerClient::processSendFromDataServer(unsigned char* packetData, int Avatar* avatar = (Avatar *) node->getLinkedData(); if (avatar->getUUID() == userUUID) { - QMetaObject::invokeMethod(&avatar->getHead().getBlendFace(), - "setModelURL", - Q_ARG(QUrl, QUrl(valueList[0]))); + QMetaObject::invokeMethod(&avatar->getHead().getFaceModel(), + "setURL", Q_ARG(QUrl, QUrl(valueList[0]))); } } } @@ -161,9 +160,8 @@ void DataServerClient::processSendFromDataServer(unsigned char* packetData, int Avatar* avatar = (Avatar *) node->getLinkedData(); if (avatar->getUUID() == userUUID) { - QMetaObject::invokeMethod(&avatar->getBody(), - "setSkeletonModelURL", - Q_ARG(QUrl, QUrl(valueList[0]))); + QMetaObject::invokeMethod(&avatar->getSkeletonModel(), "setURL", + Q_ARG(QUrl, QUrl(valueList[0]))); } } } diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 7f2afe2fa1..affc2ba355 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -297,7 +297,7 @@ Menu::Menu() : addActionToQMenuAndActionHash(avatarOptionsMenu, MenuOption::FaceMode, 0, - &appInstance->getAvatar()->getHead().getFace(), + &appInstance->getAvatar()->getHead().getVideoFace(), SLOT(cycleRenderMode())); addCheckableActionToQMenuAndActionHash(avatarOptionsMenu, MenuOption::LookAtVectors, 0, true); diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index f704449aad..a6750f820c 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -78,8 +78,8 @@ void Avatar::sendAvatarURLsMessage(const QUrl& voxelURL) { Avatar::Avatar(Node* owningNode) : AvatarData(owningNode), _head(this), - _body(this), _hand(this), + _skeletonModel(this), _ballSpringsInitialized(false), _bodyYawDelta(0.0f), _movedHandOffset(0.0f, 0.0f, 0.0f), @@ -261,6 +261,7 @@ Avatar::~Avatar() { void Avatar::init() { _head.init(); _hand.init(); + _skeletonModel.init(); _voxels.init(); _initialized = true; } @@ -415,7 +416,7 @@ void Avatar::simulate(float deltaTime, Transmitter* transmitter) { _head.setPosition(_bodyBall[ BODY_BALL_HEAD_BASE ].position); _head.setSkinColor(glm::vec3(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2])); _head.simulate(deltaTime, false); - _body.simulate(deltaTime); + _skeletonModel.simulate(deltaTime); _hand.simulate(deltaTime, false); // use speed and angular velocity to determine walking vs. standing @@ -744,18 +745,18 @@ float Avatar::getBallRenderAlpha(int ball, bool lookingInMirror) const { void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { - if (_head.getFace().isFullFrame()) { + if (_head.getVideoFace().isFullFrame()) { // Render the full-frame video float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); if (alpha > 0.0f) { - _head.getFace().render(1.0f); + _head.getVideoFace().render(1.0f); } - } else if (renderAvatarBalls || !(_voxels.getVoxelURL().isValid() || _body.isActive())) { + } else if (renderAvatarBalls || !(_voxels.getVoxelURL().isValid() || _skeletonModel.isActive())) { // Render the body as balls and cones glm::vec3 skinColor(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2]); glm::vec3 darkSkinColor(DARK_SKIN_COLOR[0], DARK_SKIN_COLOR[1], DARK_SKIN_COLOR[2]); - if (_head.getBlendFace().isActive()) { - skinColor = glm::vec3(_head.getBlendFace().computeAverageColor()); + if (_head.getFaceModel().isActive()) { + skinColor = glm::vec3(_head.getFaceModel().computeAverageColor()); const float SKIN_DARKENING = 0.9f; darkSkinColor = skinColor * SKIN_DARKENING; } @@ -780,7 +781,7 @@ void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { skinColor.g - _bodyBall[b].touchForce * 0.2f, skinColor.b - _bodyBall[b].touchForce * 0.1f); - if (b == BODY_BALL_NECK_BASE && _head.getBlendFace().isActive()) { + if (b == BODY_BALL_NECK_BASE && _head.getFaceModel().isActive()) { continue; // don't render the neck if we have a face model } @@ -815,7 +816,7 @@ void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { // Render the body's voxels and head float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); if (alpha > 0.0f) { - if (!_body.render(alpha)) { + if (!_skeletonModel.render(alpha)) { _voxels.render(false); } _head.render(alpha, false); diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index 074142cc8f..50e9eefdc0 100755 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -18,11 +18,11 @@ #include "AvatarTouch.h" #include "AvatarVoxelSystem.h" #include "Balls.h" -#include "Body.h" #include "Hand.h" #include "Head.h" #include "InterfaceConfig.h" #include "Skeleton.h" +#include "SkeletonModel.h" #include "world.h" #include "devices/SerialInterface.h" #include "devices/Transmitter.h" @@ -147,12 +147,12 @@ public: //getters bool isInitialized() const { return _initialized; } const Skeleton& getSkeleton() const { return _skeleton; } + SkeletonModel& getSkeletonModel() { return _skeletonModel; } float getHeadYawRate() const { return _head.yawRate; } const glm::vec3& getHeadJointPosition() const { return _skeleton.joint[ AVATAR_JOINT_HEAD_BASE ].position; } float getScale() const { return _scale; } const glm::vec3& getVelocity() const { return _velocity; } Head& getHead() { return _head; } - Body& getBody() { return _body; } Hand& getHand() { return _hand; } glm::quat getOrientation() const; glm::quat getWorldAlignedOrientation() const; @@ -198,9 +198,9 @@ protected: }; Head _head; - Body _body; Hand _hand; Skeleton _skeleton; + SkeletonModel _skeletonModel; bool _ballSpringsInitialized; float _bodyYawDelta; glm::vec3 _movedHandOffset; diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp deleted file mode 100644 index 8f43752736..0000000000 --- a/interface/src/avatar/BlendFace.cpp +++ /dev/null @@ -1,430 +0,0 @@ -// -// BlendFace.cpp -// interface -// -// Created by Andrzej Kapolka on 9/16/13. -// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. -// - -#include - -#include - -#include "Application.h" -#include "BlendFace.h" -#include "Head.h" - -using namespace fs; -using namespace std; - -BlendFace::BlendFace(Head* owningHead) : - _owningHead(owningHead) -{ - // we may have been created in the network thread, but we live in the main thread - moveToThread(Application::getInstance()->thread()); -} - -BlendFace::~BlendFace() { - deleteGeometry(); -} - -ProgramObject BlendFace::_program; -ProgramObject BlendFace::_skinProgram; -int BlendFace::_clusterMatricesLocation; -int BlendFace::_clusterIndicesLocation; -int BlendFace::_clusterWeightsLocation; - -void BlendFace::init() { - if (!_program.isLinked()) { - switchToResourcesParentIfRequired(); - _program.addShaderFromSourceFile(QGLShader::Vertex, "resources/shaders/blendface.vert"); - _program.addShaderFromSourceFile(QGLShader::Fragment, "resources/shaders/blendface.frag"); - _program.link(); - - _program.bind(); - _program.setUniformValue("texture", 0); - _program.release(); - - _skinProgram.addShaderFromSourceFile(QGLShader::Vertex, "resources/shaders/skin_blendface.vert"); - _skinProgram.addShaderFromSourceFile(QGLShader::Fragment, "resources/shaders/blendface.frag"); - _skinProgram.link(); - - _skinProgram.bind(); - _clusterMatricesLocation = _skinProgram.uniformLocation("clusterMatrices"); - _clusterIndicesLocation = _skinProgram.attributeLocation("clusterIndices"); - _clusterWeightsLocation = _skinProgram.attributeLocation("clusterWeights"); - _skinProgram.setUniformValue("texture", 0); - _skinProgram.release(); - } -} - -void BlendFace::reset() { - _resetStates = true; -} - -const glm::vec3 MODEL_TRANSLATION(0.0f, -60.0f, 40.0f); // temporary fudge factor -const float MODEL_SCALE = 0.0006f; - -void BlendFace::simulate(float deltaTime) { - if (!isActive()) { - return; - } - - // set up world vertices on first simulate after load - const FBXGeometry& geometry = _geometry->getFBXGeometry(); - if (_meshStates.isEmpty()) { - QVector vertices; - foreach (const FBXJoint& joint, geometry.joints) { - JointState state; - state.rotation = joint.rotation; - _jointStates.append(state); - } - foreach (const FBXMesh& mesh, geometry.meshes) { - MeshState state; - state.clusterMatrices.resize(mesh.clusters.size()); - if (mesh.springiness > 0.0f) { - state.worldSpaceVertices.resize(mesh.vertices.size()); - state.vertexVelocities.resize(mesh.vertices.size()); - state.worldSpaceNormals.resize(mesh.vertices.size()); - } - _meshStates.append(state); - } - _resetStates = true; - } - - const Skeleton& skeleton = static_cast(_owningHead->_owningAvatar)->getSkeleton(); - glm::quat orientation = skeleton.joint[AVATAR_JOINT_NECK_BASE].absoluteRotation; - glm::vec3 scale = glm::vec3(-1.0f, 1.0f, -1.0f) * _owningHead->getScale() * MODEL_SCALE; - glm::vec3 offset = MODEL_TRANSLATION - geometry.neckPivot; - glm::mat4 baseTransform = glm::translate(skeleton.joint[AVATAR_JOINT_NECK_BASE].position) * glm::mat4_cast(orientation) * - glm::scale(scale) * glm::translate(offset); - - // update the world space transforms for all joints - for (int i = 0; i < _jointStates.size(); i++) { - JointState& state = _jointStates[i]; - const FBXJoint& joint = geometry.joints.at(i); - if (joint.parentIndex == -1) { - state.transform = baseTransform * geometry.offset * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; - - } else { - if (i == geometry.neckJointIndex) { - // get the rotation axes in joint space and use them to adjust the rotation - glm::mat3 axes = glm::mat3_cast(orientation); - glm::mat3 inverse = glm::inverse(glm::mat3(_jointStates[joint.parentIndex].transform * - joint.preRotation * glm::mat4_cast(joint.rotation))); - state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * - glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * - glm::angleAxis(_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; - - } else if (i == geometry.leftEyeJointIndex || i == geometry.rightEyeJointIndex) { - // likewise with the lookat position - glm::mat4 inverse = glm::inverse(_jointStates[joint.parentIndex].transform * - joint.preRotation * glm::mat4_cast(joint.rotation)); - glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getOrientation() * IDENTITY_FRONT, 0.0f)); - glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getLookAtPosition() + - _owningHead->getSaccade(), 1.0f)); - state.rotation = rotationBetween(front, lookAt) * joint.rotation; - } - state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; - } - } - - for (int i = 0; i < _meshStates.size(); i++) { - MeshState& state = _meshStates[i]; - const FBXMesh& mesh = geometry.meshes.at(i); - for (int j = 0; j < mesh.clusters.size(); j++) { - const FBXCluster& cluster = mesh.clusters.at(j); - state.clusterMatrices[j] = _jointStates[cluster.jointIndex].transform * cluster.inverseBindMatrix; - } - int vertexCount = state.worldSpaceVertices.size(); - if (vertexCount == 0) { - continue; - } - glm::vec3* destVertices = state.worldSpaceVertices.data(); - glm::vec3* destVelocities = state.vertexVelocities.data(); - glm::vec3* destNormals = state.worldSpaceNormals.data(); - - const glm::vec3* sourceVertices = mesh.vertices.constData(); - if (!mesh.blendshapes.isEmpty()) { - _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); - memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3)); - - // blend in each coefficient - const vector& coefficients = _owningHead->getBlendshapeCoefficients(); - for (int j = 0; j < coefficients.size(); j++) { - float coefficient = coefficients[j]; - if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) { - continue; - } - const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData(); - for (const int* index = mesh.blendshapes[j].indices.constData(), - *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++) { - _blendedVertices[*index] += *vertex * coefficient; - } - } - sourceVertices = _blendedVertices.constData(); - } - glm::mat4 transform; - if (mesh.clusters.size() > 1) { - _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); - - // skin each vertex - const glm::vec4* clusterIndices = mesh.clusterIndices.constData(); - const glm::vec4* clusterWeights = mesh.clusterWeights.constData(); - for (int j = 0; j < vertexCount; j++) { - _blendedVertices[j] = - glm::vec3(state.clusterMatrices[clusterIndices[j][0]] * - glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][0] + - glm::vec3(state.clusterMatrices[clusterIndices[j][1]] * - glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][1] + - glm::vec3(state.clusterMatrices[clusterIndices[j][2]] * - glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][2] + - glm::vec3(state.clusterMatrices[clusterIndices[j][3]] * - glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][3]; - } - sourceVertices = _blendedVertices.constData(); - - } else { - transform = state.clusterMatrices[0]; - } - if (_resetStates) { - for (int j = 0; j < vertexCount; j++) { - destVertices[j] = glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f)); - destVelocities[j] = glm::vec3(); - } - } else { - const float SPRINGINESS_MULTIPLIER = 200.0f; - const float DAMPING = 5.0f; - for (int j = 0; j < vertexCount; j++) { - destVelocities[j] += ((glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f)) - destVertices[j]) * - mesh.springiness * SPRINGINESS_MULTIPLIER - destVelocities[j] * DAMPING) * deltaTime; - destVertices[j] += destVelocities[j] * deltaTime; - } - } - for (int j = 0; j < vertexCount; j++) { - destNormals[j] = glm::vec3(); - - const glm::vec3& middle = destVertices[j]; - for (QVarLengthArray, 4>::const_iterator connection = mesh.vertexConnections.at(j).constBegin(); - connection != mesh.vertexConnections.at(j).constEnd(); connection++) { - destNormals[j] += glm::normalize(glm::cross(destVertices[connection->second] - middle, - destVertices[connection->first] - middle)); - } - } - } - _resetStates = false; -} - -bool BlendFace::render(float alpha) { - if (_meshStates.isEmpty()) { - return false; - } - - // set up blended buffer ids on first render after load/simulate - const FBXGeometry& geometry = _geometry->getFBXGeometry(); - const QVector& networkMeshes = _geometry->getMeshes(); - if (_blendedVertexBufferIDs.isEmpty()) { - foreach (const FBXMesh& mesh, geometry.meshes) { - GLuint id = 0; - if (!mesh.blendshapes.isEmpty() || mesh.springiness > 0.0f) { - glGenBuffers(1, &id); - glBindBuffer(GL_ARRAY_BUFFER, id); - glBufferData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3), - NULL, GL_DYNAMIC_DRAW); - glBindBuffer(GL_ARRAY_BUFFER, 0); - } - _blendedVertexBufferIDs.append(id); - - QVector > dilated; - dilated.resize(mesh.parts.size()); - _dilatedTextures.append(dilated); - } - } - - glEnableClientState(GL_VERTEX_ARRAY); - glEnableClientState(GL_NORMAL_ARRAY); - glEnableClientState(GL_TEXTURE_COORD_ARRAY); - - glDisable(GL_COLOR_MATERIAL); - - for (int i = 0; i < networkMeshes.size(); i++) { - const NetworkMesh& networkMesh = networkMeshes.at(i); - glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, networkMesh.indexBufferID); - - const FBXMesh& mesh = geometry.meshes.at(i); - int vertexCount = mesh.vertices.size(); - - glBindBuffer(GL_ARRAY_BUFFER, networkMesh.vertexBufferID); - - const MeshState& state = _meshStates.at(i); - if (state.worldSpaceVertices.isEmpty()) { - if (state.clusterMatrices.size() > 1) { - _skinProgram.bind(); - glUniformMatrix4fvARB(_clusterMatricesLocation, state.clusterMatrices.size(), false, - (const float*)state.clusterMatrices.constData()); - int offset = vertexCount * sizeof(glm::vec2) + (mesh.blendshapes.isEmpty() ? - vertexCount * 2 * sizeof(glm::vec3) : 0); - _skinProgram.setAttributeBuffer(_clusterIndicesLocation, GL_FLOAT, offset, 4); - _skinProgram.setAttributeBuffer(_clusterWeightsLocation, GL_FLOAT, - offset + vertexCount * sizeof(glm::vec4), 4); - _skinProgram.enableAttributeArray(_clusterIndicesLocation); - _skinProgram.enableAttributeArray(_clusterWeightsLocation); - - } else { - glPushMatrix(); - glMultMatrixf((const GLfloat*)&state.clusterMatrices[0]); - _program.bind(); - } - } else { - _program.bind(); - } - - if (mesh.blendshapes.isEmpty() && mesh.springiness == 0.0f) { - glTexCoordPointer(2, GL_FLOAT, 0, (void*)(vertexCount * 2 * sizeof(glm::vec3))); - - } else { - glTexCoordPointer(2, GL_FLOAT, 0, 0); - glBindBuffer(GL_ARRAY_BUFFER, _blendedVertexBufferIDs.at(i)); - - if (!state.worldSpaceVertices.isEmpty()) { - glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), state.worldSpaceVertices.constData()); - glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3), - vertexCount * sizeof(glm::vec3), state.worldSpaceNormals.constData()); - - } else { - _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); - _blendedNormals.resize(_blendedVertices.size()); - memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3)); - memcpy(_blendedNormals.data(), mesh.normals.constData(), vertexCount * sizeof(glm::vec3)); - - // blend in each coefficient - const vector& coefficients = _owningHead->getBlendshapeCoefficients(); - for (int j = 0; j < coefficients.size(); j++) { - float coefficient = coefficients[j]; - if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) { - continue; - } - const float NORMAL_COEFFICIENT_SCALE = 0.01f; - float normalCoefficient = coefficient * NORMAL_COEFFICIENT_SCALE; - const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData(); - const glm::vec3* normal = mesh.blendshapes[j].normals.constData(); - for (const int* index = mesh.blendshapes[j].indices.constData(), - *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++, normal++) { - _blendedVertices[*index] += *vertex * coefficient; - _blendedNormals[*index] += *normal * normalCoefficient; - } - } - - glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), _blendedVertices.constData()); - glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3), - vertexCount * sizeof(glm::vec3), _blendedNormals.constData()); - } - } - glVertexPointer(3, GL_FLOAT, 0, 0); - glNormalPointer(GL_FLOAT, 0, (void*)(vertexCount * sizeof(glm::vec3))); - - qint64 offset = 0; - for (int j = 0; j < networkMesh.parts.size(); j++) { - const NetworkMeshPart& networkPart = networkMesh.parts.at(j); - const FBXMeshPart& part = mesh.parts.at(j); - - // apply material properties - glm::vec4 diffuse = glm::vec4(part.diffuseColor, alpha); - glm::vec4 specular = glm::vec4(part.specularColor, alpha); - glMaterialfv(GL_FRONT, GL_AMBIENT, (const float*)&diffuse); - glMaterialfv(GL_FRONT, GL_DIFFUSE, (const float*)&diffuse); - glMaterialfv(GL_FRONT, GL_SPECULAR, (const float*)&specular); - glMaterialf(GL_FRONT, GL_SHININESS, part.shininess); - - Texture* texture = networkPart.diffuseTexture.data(); - if (mesh.isEye) { - if (texture != NULL) { - texture = (_dilatedTextures[i][j] = static_cast(texture)->getDilatedTexture( - _owningHead->getPupilDilation())).data(); - } - } - glBindTexture(GL_TEXTURE_2D, texture == NULL ? Application::getInstance()->getTextureCache()->getWhiteTextureID() : - texture->getID()); - - glDrawRangeElementsEXT(GL_QUADS, 0, vertexCount - 1, part.quadIndices.size(), GL_UNSIGNED_INT, (void*)offset); - offset += part.quadIndices.size() * sizeof(int); - glDrawRangeElementsEXT(GL_TRIANGLES, 0, vertexCount - 1, part.triangleIndices.size(), - GL_UNSIGNED_INT, (void*)offset); - offset += part.triangleIndices.size() * sizeof(int); - } - - if (state.worldSpaceVertices.isEmpty()) { - if (state.clusterMatrices.size() > 1) { - _skinProgram.disableAttributeArray(_clusterIndicesLocation); - _skinProgram.disableAttributeArray(_clusterWeightsLocation); - _skinProgram.release(); - - } else { - glPopMatrix(); - _program.release(); - } - } else { - _program.release(); - } - } - - // deactivate vertex arrays after drawing - glDisableClientState(GL_NORMAL_ARRAY); - glDisableClientState(GL_VERTEX_ARRAY); - glDisableClientState(GL_TEXTURE_COORD_ARRAY); - - // bind with 0 to switch back to normal operation - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); - glBindTexture(GL_TEXTURE_2D, 0); - - // restore all the default material settings - Application::getInstance()->setupWorldLight(*Application::getInstance()->getCamera()); - - return true; -} - -bool BlendFace::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition, bool upright) const { - if (!isActive() || _jointStates.isEmpty()) { - return false; - } - const FBXGeometry& geometry = _geometry->getFBXGeometry(); - if (geometry.leftEyeJointIndex != -1) { - const glm::mat4& transform = _jointStates[geometry.leftEyeJointIndex].transform; - firstEyePosition = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); - } - if (geometry.rightEyeJointIndex != -1) { - const glm::mat4& transform = _jointStates[geometry.rightEyeJointIndex].transform; - secondEyePosition = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); - } - return geometry.leftEyeJointIndex != -1 && geometry.rightEyeJointIndex != -1; -} - -glm::vec4 BlendFace::computeAverageColor() const { - return _geometry ? _geometry->computeAverageColor() : glm::vec4(1.0f, 1.0f, 1.0f, 1.0f); -} - -void BlendFace::setModelURL(const QUrl& url) { - // don't recreate the geometry if it's the same URL - if (_modelURL == url) { - return; - } - _modelURL = url; - - // delete our local geometry and custom textures - deleteGeometry(); - _dilatedTextures.clear(); - - _geometry = Application::getInstance()->getGeometryCache()->getGeometry(url); -} - -void BlendFace::deleteGeometry() { - foreach (GLuint id, _blendedVertexBufferIDs) { - glDeleteBuffers(1, &id); - } - _blendedVertexBufferIDs.clear(); - _jointStates.clear(); - _meshStates.clear(); -} diff --git a/interface/src/avatar/BlendFace.h b/interface/src/avatar/BlendFace.h deleted file mode 100644 index 4721031107..0000000000 --- a/interface/src/avatar/BlendFace.h +++ /dev/null @@ -1,92 +0,0 @@ -// -// BlendFace.h -// interface -// -// Created by Andrzej Kapolka on 9/16/13. -// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. -// - -#ifndef __interface__BlendFace__ -#define __interface__BlendFace__ - -#include -#include - -#include "InterfaceConfig.h" -#include "renderer/GeometryCache.h" -#include "renderer/ProgramObject.h" -#include "renderer/TextureCache.h" - -class QNetworkReply; - -class Head; - -/// A face formed from a linear mix of blendshapes according to a set of coefficients. -class BlendFace : public QObject { - Q_OBJECT - -public: - - BlendFace(Head* owningHead); - ~BlendFace(); - - bool isActive() const { return _geometry && _geometry->isLoaded(); } - - void init(); - void reset(); - void simulate(float deltaTime); - bool render(float alpha); - - Q_INVOKABLE void setModelURL(const QUrl& url); - const QUrl& getModelURL() const { return _modelURL; } - - /// Retrieve the positions of up to two eye meshes. - /// \param upright if true, retrieve the locations of the eyes in the upright position - /// \return whether or not both eye meshes were found - bool getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition, bool upright = false) const; - - /// Returns the average color of all meshes in the geometry. - glm::vec4 computeAverageColor() const; - -private: - - void deleteGeometry(); - - Head* _owningHead; - - QUrl _modelURL; - - QSharedPointer _geometry; - - class JointState { - public: - glm::quat rotation; - glm::mat4 transform; - }; - - QVector _jointStates; - - class MeshState { - public: - QVector clusterMatrices; - QVector worldSpaceVertices; - QVector vertexVelocities; - QVector worldSpaceNormals; - }; - - QVector _meshStates; - QVector _blendedVertexBufferIDs; - QVector > > _dilatedTextures; - bool _resetStates; - - QVector _blendedVertices; - QVector _blendedNormals; - - static ProgramObject _program; - static ProgramObject _skinProgram; - static int _clusterMatricesLocation; - static int _clusterIndicesLocation; - static int _clusterWeightsLocation; -}; - -#endif /* defined(__interface__BlendFace__) */ diff --git a/interface/src/avatar/Body.cpp b/interface/src/avatar/Body.cpp deleted file mode 100644 index 857edeff13..0000000000 --- a/interface/src/avatar/Body.cpp +++ /dev/null @@ -1,81 +0,0 @@ -// -// Body.cpp -// interface -// -// Created by Andrzej Kapolka on 10/17/13. -// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. -// - -#include - -#include "Application.h" -#include "Body.h" - -Body::Body(Avatar* owningAvatar) : _owningAvatar(owningAvatar) { - // we may have been created in the network thread, but we live in the main thread - moveToThread(Application::getInstance()->thread()); -} - -void Body::simulate(float deltaTime) { - if (!isActive()) { - return; - } - - // set up joint states on first simulate after load - const FBXGeometry& geometry = _skeletonGeometry->getFBXGeometry(); - if (_jointStates.isEmpty()) { - foreach (const FBXJoint& joint, geometry.joints) { - JointState state; - state.rotation = joint.rotation; - _jointStates.append(state); - } - } - - glm::quat orientation = _owningAvatar->getOrientation(); - const float MODEL_SCALE = 0.05f; - glm::vec3 scale = glm::vec3(-1.0f, 1.0f, -1.0f) * _owningAvatar->getScale() * MODEL_SCALE; - glm::mat4 baseTransform = glm::translate(_owningAvatar->getPosition()) * glm::mat4_cast(orientation) * glm::scale(scale); - - // update the world space transforms for all joints - for (int i = 0; i < _jointStates.size(); i++) { - JointState& state = _jointStates[i]; - const FBXJoint& joint = geometry.joints.at(i); - if (joint.parentIndex == -1) { - state.transform = baseTransform * geometry.offset * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; - - } else { - state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; - } - } -} - -bool Body::render(float alpha) { - if (_jointStates.isEmpty()) { - return false; - } - - glColor4f(1.0f, 1.0f, 1.0f, alpha); - - for (int i = 0; i < _jointStates.size(); i++) { - const JointState& state = _jointStates[i]; - glPushMatrix(); - glMultMatrixf((const GLfloat*)&state.transform); - - glutSolidSphere(0.2f, 10, 10); - - glPopMatrix(); - } - - return true; -} - -void Body::setSkeletonModelURL(const QUrl& url) { - // don't recreate the geometry if it's the same URL - if (_skeletonModelURL == url) { - return; - } - _skeletonModelURL = url; - _skeletonGeometry = Application::getInstance()->getGeometryCache()->getGeometry(url); -} diff --git a/interface/src/avatar/Body.h b/interface/src/avatar/Body.h deleted file mode 100644 index 5cb6790dbd..0000000000 --- a/interface/src/avatar/Body.h +++ /dev/null @@ -1,50 +0,0 @@ -// -// Body.h -// interface -// -// Created by Andrzej Kapolka on 10/17/13. -// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. -// - -#ifndef __interface__Body__ -#define __interface__Body__ - -#include -#include - -#include "renderer/GeometryCache.h" - -/// An avatar body with an arbitrary skeleton. -class Body : public QObject { - Q_OBJECT - -public: - - Body(Avatar* owningAvatar); - - bool isActive() const { return _skeletonGeometry && _skeletonGeometry->isLoaded(); } - - void simulate(float deltaTime); - bool render(float alpha); - - Q_INVOKABLE void setSkeletonModelURL(const QUrl& url); - const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; } - -private: - - Avatar* _owningAvatar; - - QUrl _skeletonModelURL; - - QSharedPointer _skeletonGeometry; - - class JointState { - public: - glm::quat rotation; - glm::mat4 transform; - }; - - QVector _jointStates; -}; - -#endif /* defined(__interface__Body__) */ diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp new file mode 100644 index 0000000000..974d68562c --- /dev/null +++ b/interface/src/avatar/FaceModel.cpp @@ -0,0 +1,55 @@ +// +// FaceModel.cpp +// interface +// +// Created by Andrzej Kapolka on 9/16/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include "Avatar.h" +#include "FaceModel.h" +#include "Head.h" + +FaceModel::FaceModel(Head* owningHead) : + _owningHead(owningHead) +{ +} + +void FaceModel::simulate(float deltaTime) { + if (!isActive()) { + return; + } + + const Skeleton& skeleton = static_cast(_owningHead->_owningAvatar)->getSkeleton(); + setTranslation(skeleton.joint[AVATAR_JOINT_NECK_BASE].position); + setRotation(skeleton.joint[AVATAR_JOINT_NECK_BASE].absoluteRotation); + const float MODEL_SCALE = 0.0006f; + setScale(glm::vec3(-1.0f, 1.0f, -1.0f) * _owningHead->getScale() * MODEL_SCALE); + const glm::vec3 MODEL_TRANSLATION(0.0f, -60.0f, 40.0f); // temporary fudge factor + setOffset(MODEL_TRANSLATION - _geometry->getFBXGeometry().neckPivot); + + setPupilDilation(_owningHead->getPupilDilation()); + setBlendshapeCoefficients(_owningHead->getBlendshapeCoefficients()); + + Model::simulate(deltaTime); +} + +void FaceModel::maybeUpdateNeckRotation(const FBXJoint& joint, JointState& state) { + // get the rotation axes in joint space and use them to adjust the rotation + glm::mat3 axes = glm::mat3_cast(getRotation()); + glm::mat3 inverse = glm::inverse(glm::mat3(_jointStates[joint.parentIndex].transform * + joint.preRotation * glm::mat4_cast(joint.rotation))); + state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * + glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * + glm::angleAxis(_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; +} + +void FaceModel::maybeUpdateEyeRotation(const FBXJoint& joint, JointState& state) { + // get the lookat position in joint space and use it to adjust the rotation + glm::mat4 inverse = glm::inverse(_jointStates[joint.parentIndex].transform * + joint.preRotation * glm::mat4_cast(joint.rotation)); + glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getOrientation() * IDENTITY_FRONT, 0.0f)); + glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getLookAtPosition() + + _owningHead->getSaccade(), 1.0f)); + state.rotation = rotationBetween(front, lookAt) * joint.rotation; +} diff --git a/interface/src/avatar/FaceModel.h b/interface/src/avatar/FaceModel.h new file mode 100644 index 0000000000..4b9cd66cca --- /dev/null +++ b/interface/src/avatar/FaceModel.h @@ -0,0 +1,39 @@ +// +// FaceModel.h +// interface +// +// Created by Andrzej Kapolka on 9/16/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__FaceModel__ +#define __interface__FaceModel__ + +#include "renderer/Model.h" + +class Head; + +/// A face formed from a linear mix of blendshapes according to a set of coefficients. +class FaceModel : public Model { + Q_OBJECT + +public: + + FaceModel(Head* owningHead); + + void simulate(float deltaTime); + +protected: + + /// Applies neck rotation based on head orientation. + virtual void maybeUpdateNeckRotation(const FBXJoint& joint, JointState& state); + + /// Applies eye rotation based on lookat position. + virtual void maybeUpdateEyeRotation(const FBXJoint& joint, JointState& state); + +private: + + Head* _owningHead; +}; + +#endif /* defined(__interface__FaceModel__) */ diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index b6be6cf1d3..57ad245032 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -83,8 +83,8 @@ Head::Head(Avatar* owningAvatar) : _mousePitch(0.f), _cameraYaw(_yaw), _isCameraMoving(false), - _face(this), - _blendFace(this) + _videoFace(this), + _faceModel(this) { if (USING_PHYSICAL_MOHAWK) { resetHairPhysics(); @@ -104,7 +104,7 @@ void Head::init() { _irisTexture = Application::getInstance()->getTextureCache()->getTexture(QUrl::fromLocalFile(IRIS_TEXTURE_FILENAME), true).staticCast(); } - _blendFace.init(); + _faceModel.init(); } void Head::reset() { @@ -115,7 +115,7 @@ void Head::reset() { resetHairPhysics(); } - _blendFace.reset(); + _faceModel.reset(); } void Head::resetHairPhysics() { @@ -237,7 +237,7 @@ void Head::simulate(float deltaTime, bool isMine) { updateHairPhysics(deltaTime); } - _blendFace.simulate(deltaTime); + _faceModel.simulate(deltaTime); } void Head::calculateGeometry() { @@ -285,7 +285,7 @@ void Head::calculateGeometry() { void Head::render(float alpha, bool isMine) { _renderAlpha = alpha; - if (!(_face.render(alpha) || _blendFace.render(alpha))) { + if (!(_videoFace.render(alpha) || _faceModel.render(alpha))) { calculateGeometry(); glEnable(GL_DEPTH_TEST); @@ -300,9 +300,9 @@ void Head::render(float alpha, bool isMine) { renderEyeBrows(); } - if (_blendFace.isActive()) { + if (_faceModel.isActive()) { // the blend face may have custom eye meshes - _blendFace.getEyePositions(_leftEyePosition, _rightEyePosition); + _faceModel.getEyePositions(_leftEyePosition, _rightEyePosition); } if (_renderLookatVectors) { diff --git a/interface/src/avatar/Head.h b/interface/src/avatar/Head.h index c8dbbf0735..6f626322be 100644 --- a/interface/src/avatar/Head.h +++ b/interface/src/avatar/Head.h @@ -18,9 +18,9 @@ #include #include "BendyLine.h" -#include "BlendFace.h" -#include "Face.h" +#include "FaceModel.h" #include "InterfaceConfig.h" +#include "VideoFace.h" #include "world.h" #include "devices/SerialInterface.h" #include "renderer/TextureCache.h" @@ -76,8 +76,8 @@ public: glm::quat getEyeRotation(const glm::vec3& eyePosition) const; - Face& getFace() { return _face; } - BlendFace& getBlendFace() { return _blendFace; } + VideoFace& getVideoFace() { return _videoFace; } + FaceModel& getFaceModel() { return _faceModel; } const bool getReturnToCenter() const { return _returnHeadToCenter; } // Do you want head to try to return to center (depends on interface detected) float getAverageLoudness() const { return _averageLoudness; } @@ -132,8 +132,8 @@ private: float _mousePitch; float _cameraYaw; bool _isCameraMoving; - Face _face; - BlendFace _blendFace; + VideoFace _videoFace; + FaceModel _faceModel; QSharedPointer _dilatedIrisTexture; @@ -154,7 +154,7 @@ private: void resetHairPhysics(); void updateHairPhysics(float deltaTime); - friend class BlendFace; + friend class FaceModel; }; #endif diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 01a6c356fc..25c202dc45 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -322,7 +322,7 @@ void MyAvatar::simulate(float deltaTime, Transmitter* transmitter) { _head.setScale(_scale); _head.setSkinColor(glm::vec3(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2])); _head.simulate(deltaTime, true); - _body.simulate(deltaTime); + _skeletonModel.simulate(deltaTime); _hand.simulate(deltaTime, true); const float WALKING_SPEED_THRESHOLD = 0.2f; @@ -391,7 +391,7 @@ void MyAvatar::updateFromGyrosAndOrWebcam(float pitchFromTouch, bool turnWithHea _head.setMousePitch(pitchFromTouch); _head.setPitch(pitchFromTouch); } - _head.getFace().clearFrame(); + _head.getVideoFace().clearFrame(); // restore rotation, lean to neutral positions const float RESTORE_RATE = 0.05f; @@ -407,7 +407,7 @@ void MyAvatar::updateFromGyrosAndOrWebcam(float pitchFromTouch, bool turnWithHea estimatedPosition = webcam->getEstimatedPosition(); // apply face data - _head.getFace().setFrameFromWebcam(); + _head.getVideoFace().setFrameFromWebcam(); // compute and store the joint rotations const JointVector& joints = webcam->getEstimatedJoints(); @@ -424,7 +424,7 @@ void MyAvatar::updateFromGyrosAndOrWebcam(float pitchFromTouch, bool turnWithHea } } } else { - _head.getFace().clearFrame(); + _head.getVideoFace().clearFrame(); } // Set the rotation of the avatar's head (as seen by others, not affecting view frustum) @@ -605,18 +605,18 @@ void MyAvatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { return; } - if (_head.getFace().isFullFrame()) { + if (_head.getVideoFace().isFullFrame()) { // Render the full-frame video float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); if (alpha > 0.0f) { - _head.getFace().render(1.0f); + _head.getVideoFace().render(1.0f); } - } else if (renderAvatarBalls || !(_voxels.getVoxelURL().isValid() || _body.isActive())) { + } else if (renderAvatarBalls || !(_voxels.getVoxelURL().isValid() || _skeletonModel.isActive())) { // Render the body as balls and cones glm::vec3 skinColor(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2]); glm::vec3 darkSkinColor(DARK_SKIN_COLOR[0], DARK_SKIN_COLOR[1], DARK_SKIN_COLOR[2]); - if (_head.getBlendFace().isActive()) { - skinColor = glm::vec3(_head.getBlendFace().computeAverageColor()); + if (_head.getFaceModel().isActive()) { + skinColor = glm::vec3(_head.getFaceModel().computeAverageColor()); const float SKIN_DARKENING = 0.9f; darkSkinColor = skinColor * SKIN_DARKENING; } @@ -650,7 +650,7 @@ void MyAvatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { alpha); } - if (b == BODY_BALL_NECK_BASE && _head.getBlendFace().isActive()) { + if (b == BODY_BALL_NECK_BASE && _head.getFaceModel().isActive()) { continue; // don't render the neck if we have a face model } @@ -686,7 +686,7 @@ void MyAvatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { // Render the body's voxels and head float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); if (alpha > 0.0f) { - if (!_body.render(alpha)) { + if (!_skeletonModel.render(alpha)) { _voxels.render(false); } _head.render(alpha, true); diff --git a/interface/src/avatar/Profile.cpp b/interface/src/avatar/Profile.cpp index 418b842e40..662175229b 100644 --- a/interface/src/avatar/Profile.cpp +++ b/interface/src/avatar/Profile.cpp @@ -44,17 +44,15 @@ void Profile::setUUID(const QUuid& uuid) { void Profile::setFaceModelURL(const QUrl& faceModelURL) { _faceModelURL = faceModelURL; - QMetaObject::invokeMethod(&Application::getInstance()->getAvatar()->getHead().getBlendFace(), - "setModelURL", - Q_ARG(QUrl, _faceModelURL)); + QMetaObject::invokeMethod(&Application::getInstance()->getAvatar()->getHead().getFaceModel(), + "setURL", Q_ARG(QUrl, _faceModelURL)); } void Profile::setSkeletonModelURL(const QUrl& skeletonModelURL) { _skeletonModelURL = skeletonModelURL; - QMetaObject::invokeMethod(&Application::getInstance()->getAvatar()->getBody(), - "setSkeletonModelURL", - Q_ARG(QUrl, _skeletonModelURL)); + QMetaObject::invokeMethod(&Application::getInstance()->getAvatar()->getSkeletonModel(), + "setURL", Q_ARG(QUrl, _skeletonModelURL)); } void Profile::updateDomain(const QString& domain) { diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp new file mode 100644 index 0000000000..5ae193135b --- /dev/null +++ b/interface/src/avatar/SkeletonModel.cpp @@ -0,0 +1,28 @@ +// +// SkeletonModel.cpp +// interface +// +// Created by Andrzej Kapolka on 10/17/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include "Avatar.h" +#include "SkeletonModel.h" + +SkeletonModel::SkeletonModel(Avatar* owningAvatar) : + _owningAvatar(owningAvatar) +{ +} + +void SkeletonModel::simulate(float deltaTime) { + if (!isActive()) { + return; + } + + setTranslation(_owningAvatar->getPosition()); + setRotation(_owningAvatar->getOrientation()); + const float MODEL_SCALE = 0.0006f; + setScale(glm::vec3(-1.0f, 1.0f, -1.0f) * _owningAvatar->getScale() * MODEL_SCALE); + + Model::simulate(deltaTime); +} diff --git a/interface/src/avatar/SkeletonModel.h b/interface/src/avatar/SkeletonModel.h new file mode 100644 index 0000000000..ea25058a11 --- /dev/null +++ b/interface/src/avatar/SkeletonModel.h @@ -0,0 +1,31 @@ +// +// SkeletonModel.h +// interface +// +// Created by Andrzej Kapolka on 10/17/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__SkeletonModel__ +#define __interface__SkeletonModel__ + +#include "renderer/Model.h" + +class Avatar; + +/// A skeleton loaded from a model. +class SkeletonModel : public Model { + Q_OBJECT + +public: + + SkeletonModel(Avatar* owningAvatar); + + void simulate(float deltaTime); + +private: + + Avatar* _owningAvatar; +}; + +#endif /* defined(__interface__SkeletonModel__) */ diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/VideoFace.cpp similarity index 95% rename from interface/src/avatar/Face.cpp rename to interface/src/avatar/VideoFace.cpp index a8e09f238e..13b9a09440 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/VideoFace.cpp @@ -1,5 +1,5 @@ // -// Face.cpp +// VideoFace.cpp // interface // // Created by Andrzej Kapolka on 7/11/13. @@ -16,26 +16,26 @@ #include "Application.h" #include "Avatar.h" #include "Head.h" -#include "Face.h" +#include "VideoFace.h" #include "renderer/ProgramObject.h" using namespace cv; -bool Face::_initialized = false; -ProgramObject Face::_videoProgram; -Face::Locations Face::_videoProgramLocations; -ProgramObject Face::_texturedProgram; -Face::Locations Face::_texturedProgramLocations; -GLuint Face::_vboID; -GLuint Face::_iboID; +bool VideoFace::_initialized = false; +ProgramObject VideoFace::_videoProgram; +VideoFace::Locations VideoFace::_videoProgramLocations; +ProgramObject VideoFace::_texturedProgram; +VideoFace::Locations VideoFace::_texturedProgramLocations; +GLuint VideoFace::_vboID; +GLuint VideoFace::_iboID; -Face::Face(Head* owningHead) : _owningHead(owningHead), _renderMode(MESH), +VideoFace::VideoFace(Head* owningHead) : _owningHead(owningHead), _renderMode(MESH), _colorTextureID(0), _depthTextureID(0), _colorCodec(), _depthCodec(), _frameCount(0) { // we may have been created in the network thread, but we live in the main thread moveToThread(Application::getInstance()->thread()); } -Face::~Face() { +VideoFace::~VideoFace() { if (_colorCodec.name != 0) { vpx_codec_destroy(&_colorCodec); @@ -55,7 +55,7 @@ Face::~Face() { } } -void Face::setFrameFromWebcam() { +void VideoFace::setFrameFromWebcam() { Webcam* webcam = Application::getInstance()->getWebcam(); if (webcam->isSending()) { _colorTextureID = webcam->getColorTextureID(); @@ -68,12 +68,12 @@ void Face::setFrameFromWebcam() { } } -void Face::clearFrame() { +void VideoFace::clearFrame() { _colorTextureID = 0; _depthTextureID = 0; } -int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { +int VideoFace::processVideoMessage(unsigned char* packetData, size_t dataBytes) { unsigned char* packetPosition = packetData; int frameCount = *(uint32_t*)packetPosition; @@ -243,7 +243,7 @@ int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { return dataBytes; } -bool Face::render(float alpha) { +bool VideoFace::render(float alpha) { if (!isActive()) { return false; } @@ -404,11 +404,11 @@ bool Face::render(float alpha) { return true; } -void Face::cycleRenderMode() { +void VideoFace::cycleRenderMode() { _renderMode = (RenderMode)((_renderMode + 1) % RENDER_MODE_COUNT); } -void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) { +void VideoFace::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) { Size2f textureSize = _textureSize; if (!color.empty()) { bool generate = (_colorTextureID == 0); @@ -457,7 +457,7 @@ void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRati _textureSize = textureSize; } -void Face::destroyCodecs() { +void VideoFace::destroyCodecs() { if (_colorCodec.name != 0) { vpx_codec_destroy(&_colorCodec); _colorCodec.name = 0; @@ -468,7 +468,7 @@ void Face::destroyCodecs() { } } -void Face::loadProgram(ProgramObject& program, const QString& suffix, const char* secondTextureUniform, Locations& locations) { +void VideoFace::loadProgram(ProgramObject& program, const QString& suffix, const char* secondTextureUniform, Locations& locations) { program.addShaderFromSourceFile(QGLShader::Vertex, "resources/shaders/face" + suffix + ".vert"); program.addShaderFromSourceFile(QGLShader::Fragment, "resources/shaders/face" + suffix + ".frag"); program.link(); diff --git a/interface/src/avatar/Face.h b/interface/src/avatar/VideoFace.h similarity index 89% rename from interface/src/avatar/Face.h rename to interface/src/avatar/VideoFace.h index 20dfe74a0c..9dd42d6ed3 100644 --- a/interface/src/avatar/Face.h +++ b/interface/src/avatar/VideoFace.h @@ -1,13 +1,13 @@ // -// Face.h +// VideoFace.h // interface // // Created by Andrzej Kapolka on 7/11/13. // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // -#ifndef __interface__Face__ -#define __interface__Face__ +#ifndef __interface__VideoFace__ +#define __interface__VideoFace__ #include @@ -22,13 +22,13 @@ class ProgramObject; const float FULL_FRAME_ASPECT = 0.0f; -class Face : public QObject { +class VideoFace : public QObject { Q_OBJECT public: - Face(Head* owningHead); - ~Face(); + VideoFace(Head* owningHead); + ~VideoFace(); bool isActive() const { return _colorTextureID != 0 || _depthTextureID != 0; } bool isFullFrame() const { return isActive() && _aspectRatio == FULL_FRAME_ASPECT; } @@ -91,4 +91,4 @@ private: static GLuint _iboID; }; -#endif /* defined(__interface__Face__) */ +#endif /* defined(__interface__VideoFace__) */ diff --git a/interface/src/devices/Webcam.cpp b/interface/src/devices/Webcam.cpp index 679ad65c54..4022975435 100644 --- a/interface/src/devices/Webcam.cpp +++ b/interface/src/devices/Webcam.cpp @@ -19,7 +19,7 @@ #include "Application.h" #include "Webcam.h" -#include "avatar/Face.h" +#include "avatar/VideoFace.h" using namespace cv; using namespace std; diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 7c4820bfbc..4129e1a23f 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -420,7 +420,7 @@ const char* FACESHIFT_BLENDSHAPES[] = { "" }; -class Model { +class FBXModel { public: QByteArray name; @@ -432,10 +432,10 @@ public: }; glm::mat4 getGlobalTransform(const QMultiHash& parentMap, - const QHash& models, QString nodeID) { + const QHash& models, QString nodeID) { glm::mat4 globalTransform; while (!nodeID.isNull()) { - const Model& model = models.value(nodeID); + const FBXModel& model = models.value(nodeID); globalTransform = model.preRotation * glm::mat4_cast(model.rotation) * model.postRotation * globalTransform; QList parentIDs = parentMap.values(nodeID); @@ -484,7 +484,7 @@ public: }; void appendModelIDs(const QString& parentID, const QMultiHash& childMap, - QHash& models, QVector& modelIDs) { + QHash& models, QVector& modelIDs) { if (models.contains(parentID)) { modelIDs.append(parentID); } @@ -502,7 +502,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QVector blendshapes; QMultiHash parentMap; QMultiHash childMap; - QHash models; + QHash models; QHash clusters; QHash textureFilenames; QHash materials; @@ -692,7 +692,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::vec3 preRotation, rotation, postRotation; glm::vec3 scale = glm::vec3(1.0f, 1.0f, 1.0f); glm::vec3 scalePivot, rotationPivot; - Model model = { name }; + FBXModel model = { name }; foreach (const FBXNode& subobject, object.children) { if (subobject.name == "Properties60") { foreach (const FBXNode& property, subobject.children) { @@ -927,7 +927,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) // convert the models to joints foreach (const QString& modelID, modelIDs) { - const Model& model = models[modelID]; + const FBXModel& model = models[modelID]; FBXJoint joint; joint.parentIndex = model.parentIndex; joint.preRotation = model.preRotation; diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index e3b29bd1fe..3f86c907f7 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -14,6 +14,7 @@ #include #include +#include class FBXNode; From ea50d6f22e9d651ffa54a619fd7a3524b955bd17 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 17:49:34 -0700 Subject: [PATCH 06/27] Forgot these guys. --- interface/src/renderer/Model.cpp | 414 +++++++++++++++++++++++++++++++ interface/src/renderer/Model.h | 118 +++++++++ 2 files changed, 532 insertions(+) create mode 100644 interface/src/renderer/Model.cpp create mode 100644 interface/src/renderer/Model.h diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp new file mode 100644 index 0000000000..d236c14c65 --- /dev/null +++ b/interface/src/renderer/Model.cpp @@ -0,0 +1,414 @@ +// +// Model.cpp +// interface +// +// Created by Andrzej Kapolka on 10/18/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include + +#include "Application.h" +#include "Model.h" + +using namespace std; + +Model::Model() : + _pupilDilation(0.0f) +{ + // we may have been created in the network thread, but we live in the main thread + moveToThread(Application::getInstance()->thread()); +} + +Model::~Model() { + deleteGeometry(); +} + +ProgramObject Model::_program; +ProgramObject Model::_skinProgram; +int Model::_clusterMatricesLocation; +int Model::_clusterIndicesLocation; +int Model::_clusterWeightsLocation; + +void Model::init() { + if (!_program.isLinked()) { + switchToResourcesParentIfRequired(); + _program.addShaderFromSourceFile(QGLShader::Vertex, "resources/shaders/model.vert"); + _program.addShaderFromSourceFile(QGLShader::Fragment, "resources/shaders/model.frag"); + _program.link(); + + _program.bind(); + _program.setUniformValue("texture", 0); + _program.release(); + + _skinProgram.addShaderFromSourceFile(QGLShader::Vertex, "resources/shaders/skin_model.vert"); + _skinProgram.addShaderFromSourceFile(QGLShader::Fragment, "resources/shaders/model.frag"); + _skinProgram.link(); + + _skinProgram.bind(); + _clusterMatricesLocation = _skinProgram.uniformLocation("clusterMatrices"); + _clusterIndicesLocation = _skinProgram.attributeLocation("clusterIndices"); + _clusterWeightsLocation = _skinProgram.attributeLocation("clusterWeights"); + _skinProgram.setUniformValue("texture", 0); + _skinProgram.release(); + } +} + +void Model::reset() { + _resetStates = true; +} + +void Model::simulate(float deltaTime) { + if (!isActive()) { + return; + } + + // set up world vertices on first simulate after load + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + if (_jointStates.isEmpty()) { + QVector vertices; + foreach (const FBXJoint& joint, geometry.joints) { + JointState state; + state.rotation = joint.rotation; + _jointStates.append(state); + } + foreach (const FBXMesh& mesh, geometry.meshes) { + MeshState state; + state.clusterMatrices.resize(mesh.clusters.size()); + if (mesh.springiness > 0.0f) { + state.worldSpaceVertices.resize(mesh.vertices.size()); + state.vertexVelocities.resize(mesh.vertices.size()); + state.worldSpaceNormals.resize(mesh.vertices.size()); + } + _meshStates.append(state); + } + _resetStates = true; + } + + // create our root transform + glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * + glm::scale(_scale) * glm::translate(_offset); + + // update the world space transforms for all joints + for (int i = 0; i < _jointStates.size(); i++) { + JointState& state = _jointStates[i]; + const FBXJoint& joint = geometry.joints.at(i); + if (joint.parentIndex == -1) { + state.transform = baseTransform * geometry.offset * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + + } else { + if (i == geometry.neckJointIndex) { + maybeUpdateNeckRotation(joint, state); + + } else if (i == geometry.leftEyeJointIndex || i == geometry.rightEyeJointIndex) { + maybeUpdateEyeRotation(joint, state); + } + state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + } + } + + for (int i = 0; i < _meshStates.size(); i++) { + MeshState& state = _meshStates[i]; + const FBXMesh& mesh = geometry.meshes.at(i); + for (int j = 0; j < mesh.clusters.size(); j++) { + const FBXCluster& cluster = mesh.clusters.at(j); + state.clusterMatrices[j] = _jointStates[cluster.jointIndex].transform * cluster.inverseBindMatrix; + } + int vertexCount = state.worldSpaceVertices.size(); + if (vertexCount == 0) { + continue; + } + glm::vec3* destVertices = state.worldSpaceVertices.data(); + glm::vec3* destVelocities = state.vertexVelocities.data(); + glm::vec3* destNormals = state.worldSpaceNormals.data(); + + const glm::vec3* sourceVertices = mesh.vertices.constData(); + if (!mesh.blendshapes.isEmpty()) { + _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); + memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3)); + + // blend in each coefficient + for (int j = 0; j < _blendshapeCoefficients.size(); j++) { + float coefficient = _blendshapeCoefficients[j]; + if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) { + continue; + } + const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData(); + for (const int* index = mesh.blendshapes[j].indices.constData(), + *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++) { + _blendedVertices[*index] += *vertex * coefficient; + } + } + sourceVertices = _blendedVertices.constData(); + } + glm::mat4 transform; + if (mesh.clusters.size() > 1) { + _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); + + // skin each vertex + const glm::vec4* clusterIndices = mesh.clusterIndices.constData(); + const glm::vec4* clusterWeights = mesh.clusterWeights.constData(); + for (int j = 0; j < vertexCount; j++) { + _blendedVertices[j] = + glm::vec3(state.clusterMatrices[clusterIndices[j][0]] * + glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][0] + + glm::vec3(state.clusterMatrices[clusterIndices[j][1]] * + glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][1] + + glm::vec3(state.clusterMatrices[clusterIndices[j][2]] * + glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][2] + + glm::vec3(state.clusterMatrices[clusterIndices[j][3]] * + glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][3]; + } + sourceVertices = _blendedVertices.constData(); + + } else { + transform = state.clusterMatrices[0]; + } + if (_resetStates) { + for (int j = 0; j < vertexCount; j++) { + destVertices[j] = glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f)); + destVelocities[j] = glm::vec3(); + } + } else { + const float SPRINGINESS_MULTIPLIER = 200.0f; + const float DAMPING = 5.0f; + for (int j = 0; j < vertexCount; j++) { + destVelocities[j] += ((glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f)) - destVertices[j]) * + mesh.springiness * SPRINGINESS_MULTIPLIER - destVelocities[j] * DAMPING) * deltaTime; + destVertices[j] += destVelocities[j] * deltaTime; + } + } + for (int j = 0; j < vertexCount; j++) { + destNormals[j] = glm::vec3(); + + const glm::vec3& middle = destVertices[j]; + for (QVarLengthArray, 4>::const_iterator connection = mesh.vertexConnections.at(j).constBegin(); + connection != mesh.vertexConnections.at(j).constEnd(); connection++) { + destNormals[j] += glm::normalize(glm::cross(destVertices[connection->second] - middle, + destVertices[connection->first] - middle)); + } + } + } + _resetStates = false; +} + +bool Model::render(float alpha) { + if (_meshStates.isEmpty()) { + return false; + } + + // set up blended buffer ids on first render after load/simulate + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + const QVector& networkMeshes = _geometry->getMeshes(); + if (_blendedVertexBufferIDs.isEmpty()) { + foreach (const FBXMesh& mesh, geometry.meshes) { + GLuint id = 0; + if (!mesh.blendshapes.isEmpty() || mesh.springiness > 0.0f) { + glGenBuffers(1, &id); + glBindBuffer(GL_ARRAY_BUFFER, id); + glBufferData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3), + NULL, GL_DYNAMIC_DRAW); + glBindBuffer(GL_ARRAY_BUFFER, 0); + } + _blendedVertexBufferIDs.append(id); + + QVector > dilated; + dilated.resize(mesh.parts.size()); + _dilatedTextures.append(dilated); + } + } + + glEnableClientState(GL_VERTEX_ARRAY); + glEnableClientState(GL_NORMAL_ARRAY); + glEnableClientState(GL_TEXTURE_COORD_ARRAY); + + glDisable(GL_COLOR_MATERIAL); + + for (int i = 0; i < networkMeshes.size(); i++) { + const NetworkMesh& networkMesh = networkMeshes.at(i); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, networkMesh.indexBufferID); + + const FBXMesh& mesh = geometry.meshes.at(i); + int vertexCount = mesh.vertices.size(); + + glBindBuffer(GL_ARRAY_BUFFER, networkMesh.vertexBufferID); + + const MeshState& state = _meshStates.at(i); + if (state.worldSpaceVertices.isEmpty()) { + if (state.clusterMatrices.size() > 1) { + _skinProgram.bind(); + glUniformMatrix4fvARB(_clusterMatricesLocation, state.clusterMatrices.size(), false, + (const float*)state.clusterMatrices.constData()); + int offset = vertexCount * sizeof(glm::vec2) + (mesh.blendshapes.isEmpty() ? + vertexCount * 2 * sizeof(glm::vec3) : 0); + _skinProgram.setAttributeBuffer(_clusterIndicesLocation, GL_FLOAT, offset, 4); + _skinProgram.setAttributeBuffer(_clusterWeightsLocation, GL_FLOAT, + offset + vertexCount * sizeof(glm::vec4), 4); + _skinProgram.enableAttributeArray(_clusterIndicesLocation); + _skinProgram.enableAttributeArray(_clusterWeightsLocation); + + } else { + glPushMatrix(); + glMultMatrixf((const GLfloat*)&state.clusterMatrices[0]); + _program.bind(); + } + } else { + _program.bind(); + } + + if (mesh.blendshapes.isEmpty() && mesh.springiness == 0.0f) { + glTexCoordPointer(2, GL_FLOAT, 0, (void*)(vertexCount * 2 * sizeof(glm::vec3))); + + } else { + glTexCoordPointer(2, GL_FLOAT, 0, 0); + glBindBuffer(GL_ARRAY_BUFFER, _blendedVertexBufferIDs.at(i)); + + if (!state.worldSpaceVertices.isEmpty()) { + glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), state.worldSpaceVertices.constData()); + glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3), + vertexCount * sizeof(glm::vec3), state.worldSpaceNormals.constData()); + + } else { + _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); + _blendedNormals.resize(_blendedVertices.size()); + memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3)); + memcpy(_blendedNormals.data(), mesh.normals.constData(), vertexCount * sizeof(glm::vec3)); + + // blend in each coefficient + for (int j = 0; j < _blendshapeCoefficients.size(); j++) { + float coefficient = _blendshapeCoefficients[j]; + if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) { + continue; + } + const float NORMAL_COEFFICIENT_SCALE = 0.01f; + float normalCoefficient = coefficient * NORMAL_COEFFICIENT_SCALE; + const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData(); + const glm::vec3* normal = mesh.blendshapes[j].normals.constData(); + for (const int* index = mesh.blendshapes[j].indices.constData(), + *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++, normal++) { + _blendedVertices[*index] += *vertex * coefficient; + _blendedNormals[*index] += *normal * normalCoefficient; + } + } + + glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), _blendedVertices.constData()); + glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3), + vertexCount * sizeof(glm::vec3), _blendedNormals.constData()); + } + } + glVertexPointer(3, GL_FLOAT, 0, 0); + glNormalPointer(GL_FLOAT, 0, (void*)(vertexCount * sizeof(glm::vec3))); + + qint64 offset = 0; + for (int j = 0; j < networkMesh.parts.size(); j++) { + const NetworkMeshPart& networkPart = networkMesh.parts.at(j); + const FBXMeshPart& part = mesh.parts.at(j); + + // apply material properties + glm::vec4 diffuse = glm::vec4(part.diffuseColor, alpha); + glm::vec4 specular = glm::vec4(part.specularColor, alpha); + glMaterialfv(GL_FRONT, GL_AMBIENT, (const float*)&diffuse); + glMaterialfv(GL_FRONT, GL_DIFFUSE, (const float*)&diffuse); + glMaterialfv(GL_FRONT, GL_SPECULAR, (const float*)&specular); + glMaterialf(GL_FRONT, GL_SHININESS, part.shininess); + + Texture* texture = networkPart.diffuseTexture.data(); + if (mesh.isEye) { + if (texture != NULL) { + texture = (_dilatedTextures[i][j] = static_cast(texture)->getDilatedTexture( + _pupilDilation)).data(); + } + } + glBindTexture(GL_TEXTURE_2D, texture == NULL ? Application::getInstance()->getTextureCache()->getWhiteTextureID() : + texture->getID()); + + glDrawRangeElementsEXT(GL_QUADS, 0, vertexCount - 1, part.quadIndices.size(), GL_UNSIGNED_INT, (void*)offset); + offset += part.quadIndices.size() * sizeof(int); + glDrawRangeElementsEXT(GL_TRIANGLES, 0, vertexCount - 1, part.triangleIndices.size(), + GL_UNSIGNED_INT, (void*)offset); + offset += part.triangleIndices.size() * sizeof(int); + } + + if (state.worldSpaceVertices.isEmpty()) { + if (state.clusterMatrices.size() > 1) { + _skinProgram.disableAttributeArray(_clusterIndicesLocation); + _skinProgram.disableAttributeArray(_clusterWeightsLocation); + _skinProgram.release(); + + } else { + glPopMatrix(); + _program.release(); + } + } else { + _program.release(); + } + } + + // deactivate vertex arrays after drawing + glDisableClientState(GL_NORMAL_ARRAY); + glDisableClientState(GL_VERTEX_ARRAY); + glDisableClientState(GL_TEXTURE_COORD_ARRAY); + + // bind with 0 to switch back to normal operation + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + glBindTexture(GL_TEXTURE_2D, 0); + + // restore all the default material settings + Application::getInstance()->setupWorldLight(*Application::getInstance()->getCamera()); + + return true; +} + +bool Model::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const { + if (!isActive() || _jointStates.isEmpty()) { + return false; + } + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + if (geometry.leftEyeJointIndex != -1) { + const glm::mat4& transform = _jointStates[geometry.leftEyeJointIndex].transform; + firstEyePosition = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); + } + if (geometry.rightEyeJointIndex != -1) { + const glm::mat4& transform = _jointStates[geometry.rightEyeJointIndex].transform; + secondEyePosition = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); + } + return geometry.leftEyeJointIndex != -1 && geometry.rightEyeJointIndex != -1; +} + +void Model::setURL(const QUrl& url) { + // don't recreate the geometry if it's the same URL + if (_url == url) { + return; + } + _url = url; + + // delete our local geometry and custom textures + deleteGeometry(); + _dilatedTextures.clear(); + + _geometry = Application::getInstance()->getGeometryCache()->getGeometry(url); +} + +glm::vec4 Model::computeAverageColor() const { + return _geometry ? _geometry->computeAverageColor() : glm::vec4(1.0f, 1.0f, 1.0f, 1.0f); +} + +void Model::maybeUpdateNeckRotation(const FBXJoint& joint, JointState& state) { + // nothing by default +} + +void Model::maybeUpdateEyeRotation(const FBXJoint& joint, JointState& state) { + // nothing by default +} + +void Model::deleteGeometry() { + foreach (GLuint id, _blendedVertexBufferIDs) { + glDeleteBuffers(1, &id); + } + _blendedVertexBufferIDs.clear(); + _jointStates.clear(); + _meshStates.clear(); +} diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h new file mode 100644 index 0000000000..eb145100e2 --- /dev/null +++ b/interface/src/renderer/Model.h @@ -0,0 +1,118 @@ +// +// Model.h +// interface +// +// Created by Andrzej Kapolka on 10/18/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__Model__ +#define __interface__Model__ + +#include +#include + +#include "GeometryCache.h" +#include "InterfaceConfig.h" +#include "ProgramObject.h" +#include "TextureCache.h" + +/// A generic 3D model displaying geometry loaded from a URL. +class Model : public QObject { + Q_OBJECT + +public: + + Model(); + virtual ~Model(); + + void setTranslation(const glm::vec3& translation) { _translation = translation; } + const glm::vec3& getTranslation() const { return _translation; } + + void setRotation(const glm::quat& rotation) { _rotation = rotation; } + const glm::quat& getRotation() const { return _rotation; } + + void setScale(const glm::vec3& scale) { _scale = scale; } + const glm::vec3& getScale() const { return _scale; } + + void setOffset(const glm::vec3& offset) { _offset = offset; } + const glm::vec3& getOffset() const { return _offset; } + + void setPupilDilation(float dilation) { _pupilDilation = dilation; } + float getPupilDilation() const { return _pupilDilation; } + + void setBlendshapeCoefficients(const std::vector& coefficients) { _blendshapeCoefficients = coefficients; } + const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + + bool isActive() const { return _geometry && _geometry->isLoaded(); } + + void init(); + void reset(); + void simulate(float deltaTime); + bool render(float alpha); + + Q_INVOKABLE void setURL(const QUrl& url); + const QUrl& getURL() const { return _url; } + + /// Retrieve the positions of up to two eye meshes. + /// \return whether or not both eye meshes were found + bool getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const; + + /// Returns the average color of all meshes in the geometry. + glm::vec4 computeAverageColor() const; + +protected: + + QSharedPointer _geometry; + + class JointState { + public: + glm::quat rotation; + glm::mat4 transform; + }; + + QVector _jointStates; + + /// Gives subclasses a chance to update the neck joint rotation after its parents and before its children. + virtual void maybeUpdateNeckRotation(const FBXJoint& joint, JointState& state); + + /// Gives subclasses a chance to update an eye joint rotation after its parents and before its children. + virtual void maybeUpdateEyeRotation(const FBXJoint& joint, JointState& state); + +private: + + void deleteGeometry(); + + glm::vec3 _translation; + glm::quat _rotation; + glm::vec3 _scale; + glm::vec3 _offset; + float _pupilDilation; + std::vector _blendshapeCoefficients; + + QUrl _url; + + class MeshState { + public: + QVector clusterMatrices; + QVector worldSpaceVertices; + QVector vertexVelocities; + QVector worldSpaceNormals; + }; + + QVector _meshStates; + QVector _blendedVertexBufferIDs; + QVector > > _dilatedTextures; + bool _resetStates; + + QVector _blendedVertices; + QVector _blendedNormals; + + static ProgramObject _program; + static ProgramObject _skinProgram; + static int _clusterMatricesLocation; + static int _clusterIndicesLocation; + static int _clusterWeightsLocation; +}; + +#endif /* defined(__interface__Model__) */ From ec17b80b52e4d37b04a2c142da8ad105f78ded72 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 17:54:54 -0700 Subject: [PATCH 07/27] Adapt to the *other* kind of cluster. --- interface/src/renderer/FBXReader.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 4129e1a23f..1339e8d7a3 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -830,7 +830,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) materials.insert(object.properties.at(0).toString(), material); } else if (object.name == "Deformer") { - if (object.properties.at(2) == "Cluster") { + if ((object.properties.size() == 3 && object.properties.at(2) == "Cluster") || + (object.properties.size() == 2 && object.properties.at(1) == "Cluster")) { Cluster cluster; foreach (const FBXNode& subobject, object.children) { if (subobject.name == "Indexes") { From a6a4ceebe45428b8fead85c72a6b5301f86c4d70 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 18:03:28 -0700 Subject: [PATCH 08/27] Just check the last entry for the type. --- interface/src/renderer/FBXReader.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 1339e8d7a3..804a19a054 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -830,8 +830,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) materials.insert(object.properties.at(0).toString(), material); } else if (object.name == "Deformer") { - if ((object.properties.size() == 3 && object.properties.at(2) == "Cluster") || - (object.properties.size() == 2 && object.properties.at(1) == "Cluster")) { + if (object.properties.last() == "Cluster") { Cluster cluster; foreach (const FBXNode& subobject, object.children) { if (subobject.name == "Indexes") { @@ -847,7 +846,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } clusters.insert(object.properties.at(0).toString(), cluster); - } else if (object.properties.at(2) == "BlendShapeChannel") { + } else if (object.properties.last() == "BlendShapeChannel") { QByteArray name = object.properties.at(1).toByteArray(); name = name.left(name.indexOf('\0')); if (!blendshapeIndices.contains(name)) { From b3384d0869ee5bfd9f98fba52e67c13c1b7579dd Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 18:23:08 -0700 Subject: [PATCH 09/27] More fixes for text FBXs. --- interface/src/renderer/FBXReader.cpp | 48 ++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 804a19a054..3fb169ac07 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -368,6 +368,28 @@ glm::mat4 createMat4(const QVector& doubleVector) { doubleVector.at(12), doubleVector.at(13), doubleVector.at(14), doubleVector.at(15)); } +QVector getIntVector(const QVariantList& properties, int index) { + QVector vector = properties.at(index).value >(); + if (!vector.isEmpty()) { + return vector; + } + for (; index < properties.size(); index++) { + vector.append(properties.at(index).toInt()); + } + return vector; +} + +QVector getDoubleVector(const QVariantList& properties, int index) { + QVector vector = properties.at(index).value >(); + if (!vector.isEmpty()) { + return vector; + } + for (; index < properties.size(); index++) { + vector.append(properties.at(index).toDouble()); + } + return vector; +} + const char* FACESHIFT_BLENDSHAPES[] = { "EyeBlink_L", "EyeBlink_R", @@ -552,19 +574,19 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QVector materials; foreach (const FBXNode& data, object.children) { if (data.name == "Vertices") { - mesh.vertices = createVec3Vector(data.properties.at(0).value >()); + mesh.vertices = createVec3Vector(getDoubleVector(data.properties, 0)); } else if (data.name == "PolygonVertexIndex") { - polygonIndices = data.properties.at(0).value >(); + polygonIndices = getIntVector(data.properties, 0); } else if (data.name == "LayerElementNormal") { bool byVertex = false; foreach (const FBXNode& subdata, data.children) { if (subdata.name == "Normals") { - normals = createVec3Vector(subdata.properties.at(0).value >()); + normals = createVec3Vector(getDoubleVector(subdata.properties, 0)); } else if (subdata.name == "NormalsIndex") { - normalIndices = subdata.properties.at(0).value >(); + normalIndices = getIntVector(subdata.properties, 0); } else if (subdata.name == "MappingInformationType" && subdata.properties.at(0) == "ByVertice") { @@ -577,16 +599,16 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } else if (data.name == "LayerElementUV" && data.properties.at(0).toInt() == 0) { foreach (const FBXNode& subdata, data.children) { if (subdata.name == "UV") { - texCoords = createVec2Vector(subdata.properties.at(0).value >()); + texCoords = createVec2Vector(getDoubleVector(subdata.properties, 0)); } else if (subdata.name == "UVIndex") { - texCoordIndices = subdata.properties.at(0).value >(); + texCoordIndices = getIntVector(subdata.properties, 0); } } } else if (data.name == "LayerElementMaterial") { foreach (const FBXNode& subdata, data.children) { if (subdata.name == "Materials") { - materials = subdata.properties.at(0).value >(); + materials = getIntVector(subdata.properties, 0); } } } @@ -661,15 +683,15 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) foreach (const FBXNode& data, object.children) { if (data.name == "Indexes") { - extracted.blendshape.indices = data.properties.at(0).value >(); + extracted.blendshape.indices = getIntVector(data.properties, 0); } else if (data.name == "Vertices") { extracted.blendshape.vertices = createVec3Vector( - data.properties.at(0).value >()); + getDoubleVector(data.properties, 0)); } else if (data.name == "Normals") { extracted.blendshape.normals = createVec3Vector( - data.properties.at(0).value >()); + getDoubleVector(data.properties, 0)); } } @@ -834,13 +856,13 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) Cluster cluster; foreach (const FBXNode& subobject, object.children) { if (subobject.name == "Indexes") { - cluster.indices = subobject.properties.at(0).value >(); + cluster.indices = getIntVector(subobject.properties, 0); } else if (subobject.name == "Weights") { - cluster.weights = subobject.properties.at(0).value >(); + cluster.weights = getDoubleVector(subobject.properties, 0); } else if (subobject.name == "TransformLink") { - QVector values = subobject.properties.at(0).value >(); + QVector values = getDoubleVector(subobject.properties, 0); cluster.transformLink = createMat4(values); } } From b294f5b13e98a16d863bbfcd1c805e1ebb0590c2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 18:43:07 -0700 Subject: [PATCH 10/27] Fix for weird links in Blender exports. --- interface/src/renderer/FBXReader.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 3fb169ac07..47ab148937 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -446,11 +446,11 @@ class FBXModel { public: QByteArray name; + int parentIndex; + glm::mat4 preRotation; glm::quat rotation; glm::mat4 postRotation; - - int parentIndex; }; glm::mat4 getGlobalTransform(const QMultiHash& parentMap, @@ -513,8 +513,11 @@ void appendModelIDs(const QString& parentID, const QMultiHash& int parentIndex = modelIDs.size() - 1; foreach (const QString& childID, childMap.values(parentID)) { if (models.contains(childID)) { - models[childID].parentIndex = parentIndex; - appendModelIDs(childID, childMap, models, modelIDs); + FBXModel& model = models[childID]; + if (model.parentIndex == -1) { + model.parentIndex = parentIndex; + appendModelIDs(childID, childMap, models, modelIDs); + } } } } @@ -714,7 +717,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::vec3 preRotation, rotation, postRotation; glm::vec3 scale = glm::vec3(1.0f, 1.0f, 1.0f); glm::vec3 scalePivot, rotationPivot; - FBXModel model = { name }; + FBXModel model = { name, -1 }; foreach (const FBXNode& subobject, object.children) { if (subobject.name == "Properties60") { foreach (const FBXNode& property, subobject.children) { From 871f97bd81becd4d6d78d2eeb36d115d3d73bb11 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 19:06:56 -0700 Subject: [PATCH 11/27] The Blender FBX has models that are also meshes. --- interface/src/renderer/FBXReader.cpp | 236 ++++++++++++++------------- 1 file changed, 122 insertions(+), 114 deletions(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 47ab148937..4ba145150b 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -522,6 +522,123 @@ void appendModelIDs(const QString& parentID, const QMultiHash& } } +FBXMesh extractMesh(const FBXNode& object) { + FBXMesh mesh; + + QVector polygonIndices; + QVector normals; + QVector normalIndices; + QVector texCoords; + QVector texCoordIndices; + QVector materials; + foreach (const FBXNode& data, object.children) { + if (data.name == "Vertices") { + mesh.vertices = createVec3Vector(getDoubleVector(data.properties, 0)); + + } else if (data.name == "PolygonVertexIndex") { + polygonIndices = getIntVector(data.properties, 0); + + } else if (data.name == "LayerElementNormal") { + bool byVertex = false; + foreach (const FBXNode& subdata, data.children) { + if (subdata.name == "Normals") { + normals = createVec3Vector(getDoubleVector(subdata.properties, 0)); + + } else if (subdata.name == "NormalsIndex") { + normalIndices = getIntVector(subdata.properties, 0); + + } else if (subdata.name == "MappingInformationType" && + subdata.properties.at(0) == "ByVertice") { + byVertex = true; + } + } + if (byVertex) { + mesh.normals = normals; + } + } else if (data.name == "LayerElementUV" && data.properties.at(0).toInt() == 0) { + foreach (const FBXNode& subdata, data.children) { + if (subdata.name == "UV") { + texCoords = createVec2Vector(getDoubleVector(subdata.properties, 0)); + + } else if (subdata.name == "UVIndex") { + texCoordIndices = getIntVector(subdata.properties, 0); + } + } + } else if (data.name == "LayerElementMaterial") { + foreach (const FBXNode& subdata, data.children) { + if (subdata.name == "Materials") { + materials = getIntVector(subdata.properties, 0); + } + } + } + } + + // convert normals from per-index to per-vertex if necessary + if (mesh.normals.isEmpty()) { + mesh.normals.resize(mesh.vertices.size()); + if (normalIndices.isEmpty()) { + for (int i = 0, n = polygonIndices.size(); i < n; i++) { + int index = polygonIndices.at(i); + mesh.normals[index < 0 ? (-index - 1) : index] = normals.at(i); + } + } else { + for (int i = 0, n = polygonIndices.size(); i < n; i++) { + int index = polygonIndices.at(i); + int normalIndex = normalIndices.at(i); + if (normalIndex >= 0) { + mesh.normals[index < 0 ? (-index - 1) : index] = normals.at(normalIndex); + } + } + } + } + + // same with the tex coords + if (!texCoordIndices.isEmpty()) { + mesh.texCoords.resize(mesh.vertices.size()); + for (int i = 0, n = polygonIndices.size(); i < n; i++) { + int index = polygonIndices.at(i); + int texCoordIndex = texCoordIndices.at(i); + if (texCoordIndex >= 0) { + mesh.texCoords[index < 0 ? (-index - 1) : index] = texCoords.at(texCoordIndex); + } + } + } + + // convert the polygons to quads and triangles + int polygonIndex = 0; + for (const int* beginIndex = polygonIndices.constData(), *end = beginIndex + polygonIndices.size(); + beginIndex != end; polygonIndex++) { + const int* endIndex = beginIndex; + while (*endIndex++ >= 0); + + int materialIndex = (polygonIndex < materials.size()) ? materials.at(polygonIndex) : 0; + mesh.parts.resize(max(mesh.parts.size(), materialIndex + 1)); + FBXMeshPart& part = mesh.parts[materialIndex]; + + if (endIndex - beginIndex == 4) { + part.quadIndices.append(*beginIndex++); + part.quadIndices.append(*beginIndex++); + part.quadIndices.append(*beginIndex++); + part.quadIndices.append(-*beginIndex++ - 1); + + } else { + for (const int* nextIndex = beginIndex + 1;; ) { + part.triangleIndices.append(*beginIndex); + part.triangleIndices.append(*nextIndex++); + if (*nextIndex >= 0) { + part.triangleIndices.append(*nextIndex); + } else { + part.triangleIndices.append(-*nextIndex - 1); + break; + } + } + beginIndex = endIndex; + } + } + + return mesh; +} + FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) { QHash meshes; QVector blendshapes; @@ -567,119 +684,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) foreach (const FBXNode& object, child.children) { if (object.name == "Geometry") { if (object.properties.at(2) == "Mesh") { - FBXMesh mesh; - - QVector polygonIndices; - QVector normals; - QVector normalIndices; - QVector texCoords; - QVector texCoordIndices; - QVector materials; - foreach (const FBXNode& data, object.children) { - if (data.name == "Vertices") { - mesh.vertices = createVec3Vector(getDoubleVector(data.properties, 0)); - - } else if (data.name == "PolygonVertexIndex") { - polygonIndices = getIntVector(data.properties, 0); - - } else if (data.name == "LayerElementNormal") { - bool byVertex = false; - foreach (const FBXNode& subdata, data.children) { - if (subdata.name == "Normals") { - normals = createVec3Vector(getDoubleVector(subdata.properties, 0)); - - } else if (subdata.name == "NormalsIndex") { - normalIndices = getIntVector(subdata.properties, 0); - - } else if (subdata.name == "MappingInformationType" && - subdata.properties.at(0) == "ByVertice") { - byVertex = true; - } - } - if (byVertex) { - mesh.normals = normals; - } - } else if (data.name == "LayerElementUV" && data.properties.at(0).toInt() == 0) { - foreach (const FBXNode& subdata, data.children) { - if (subdata.name == "UV") { - texCoords = createVec2Vector(getDoubleVector(subdata.properties, 0)); - - } else if (subdata.name == "UVIndex") { - texCoordIndices = getIntVector(subdata.properties, 0); - } - } - } else if (data.name == "LayerElementMaterial") { - foreach (const FBXNode& subdata, data.children) { - if (subdata.name == "Materials") { - materials = getIntVector(subdata.properties, 0); - } - } - } - } - - // convert normals from per-index to per-vertex if necessary - if (mesh.normals.isEmpty()) { - mesh.normals.resize(mesh.vertices.size()); - if (normalIndices.isEmpty()) { - for (int i = 0, n = polygonIndices.size(); i < n; i++) { - int index = polygonIndices.at(i); - mesh.normals[index < 0 ? (-index - 1) : index] = normals.at(i); - } - } else { - for (int i = 0, n = polygonIndices.size(); i < n; i++) { - int index = polygonIndices.at(i); - int normalIndex = normalIndices.at(i); - if (normalIndex >= 0) { - mesh.normals[index < 0 ? (-index - 1) : index] = normals.at(normalIndex); - } - } - } - } - - // same with the tex coords - if (!texCoordIndices.isEmpty()) { - mesh.texCoords.resize(mesh.vertices.size()); - for (int i = 0, n = polygonIndices.size(); i < n; i++) { - int index = polygonIndices.at(i); - int texCoordIndex = texCoordIndices.at(i); - if (texCoordIndex >= 0) { - mesh.texCoords[index < 0 ? (-index - 1) : index] = texCoords.at(texCoordIndex); - } - } - } - - // convert the polygons to quads and triangles - int polygonIndex = 0; - for (const int* beginIndex = polygonIndices.constData(), *end = beginIndex + polygonIndices.size(); - beginIndex != end; polygonIndex++) { - const int* endIndex = beginIndex; - while (*endIndex++ >= 0); - - int materialIndex = (polygonIndex < materials.size()) ? materials.at(polygonIndex) : 0; - mesh.parts.resize(max(mesh.parts.size(), materialIndex + 1)); - FBXMeshPart& part = mesh.parts[materialIndex]; - - if (endIndex - beginIndex == 4) { - part.quadIndices.append(*beginIndex++); - part.quadIndices.append(*beginIndex++); - part.quadIndices.append(*beginIndex++); - part.quadIndices.append(-*beginIndex++ - 1); - - } else { - for (const int* nextIndex = beginIndex + 1;; ) { - part.triangleIndices.append(*beginIndex); - part.triangleIndices.append(*nextIndex++); - if (*nextIndex >= 0) { - part.triangleIndices.append(*nextIndex); - } else { - part.triangleIndices.append(-*nextIndex - 1); - break; - } - } - beginIndex = endIndex; - } - } - meshes.insert(object.properties.at(0).toString(), mesh); + meshes.insert(object.properties.at(0).toString(), extractMesh(object)); } else { // object.properties.at(2) == "Shape" ExtractedBlendshape extracted = { object.properties.at(0).toString() }; @@ -809,6 +814,9 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } } } + } else if (subobject.name == "Vertices") { + // it's a mesh as well as a model + meshes.insert(object.properties.at(0).toString(), extractMesh(object)); } } // see FBX documentation, http://download.autodesk.com/us/fbx/20112/FBX_SDK_HELP/index.html @@ -986,7 +994,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) FBXMesh& mesh = it.value(); // accumulate local transforms - QString modelID = parentMap.value(it.key()); + QString modelID = models.contains(it.key()) ? it.key() : parentMap.value(it.key()); mesh.springiness = springs.value(models.value(modelID).name, defaultSpring).toFloat(); glm::mat4 modelTransform = getGlobalTransform(parentMap, models, modelID); From 7ddac20c62ee4aeec2b17c928182652b597e31f3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 18 Oct 2013 19:18:48 -0700 Subject: [PATCH 12/27] Allow more bones. The model I'm testing with has about 60. --- interface/resources/shaders/skin_model.vert | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/resources/shaders/skin_model.vert b/interface/resources/shaders/skin_model.vert index af2d0103a9..d968d17df0 100644 --- a/interface/resources/shaders/skin_model.vert +++ b/interface/resources/shaders/skin_model.vert @@ -8,7 +8,7 @@ // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // -const int MAX_CLUSTERS = 32; +const int MAX_CLUSTERS = 64; const int INDICES_PER_VERTEX = 4; uniform mat4 clusterMatrices[MAX_CLUSTERS]; From 49acca3587bf7ae2a62a5f4f4bb74f5244f70b2a Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 21 Oct 2013 11:20:27 -0700 Subject: [PATCH 13/27] Fixes to load my test model. --- interface/src/renderer/FBXReader.cpp | 139 +++++++++++---------------- 1 file changed, 57 insertions(+), 82 deletions(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 4ba145150b..ef760ddd56 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -390,6 +390,11 @@ QVector getDoubleVector(const QVariantList& properties, int index) { return vector; } +glm::vec3 getVec3(const QVariantList& properties, int index) { + return glm::vec3(properties.at(index).value(), properties.at(index + 1).value(), + properties.at(index + 2).value()); +} + const char* FACESHIFT_BLENDSHAPES[] = { "EyeBlink_L", "EyeBlink_R", @@ -724,93 +729,45 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::vec3 scalePivot, rotationPivot; FBXModel model = { name, -1 }; foreach (const FBXNode& subobject, object.children) { + bool properties = false; + QByteArray propertyName; + int index; if (subobject.name == "Properties60") { - foreach (const FBXNode& property, subobject.children) { - if (property.name == "Property") { - if (property.properties.at(0) == "Lcl Translation") { - translation = glm::vec3(property.properties.at(3).value(), - property.properties.at(4).value(), - property.properties.at(5).value()); - - } else if (property.properties.at(0) == "RotationOffset") { - rotationOffset = glm::vec3(property.properties.at(3).value(), - property.properties.at(4).value(), - property.properties.at(5).value()); - - } else if (property.properties.at(0) == "RotationPivot") { - rotationPivot = glm::vec3(property.properties.at(3).value(), - property.properties.at(4).value(), - property.properties.at(5).value()); - - } else if (property.properties.at(0) == "PreRotation") { - preRotation = glm::vec3(property.properties.at(3).value(), - property.properties.at(4).value(), - property.properties.at(5).value()); - - } else if (property.properties.at(0) == "Lcl Rotation") { - rotation = glm::vec3(property.properties.at(3).value(), - property.properties.at(4).value(), - property.properties.at(5).value()); - - } else if (property.properties.at(0) == "PostRotation") { - postRotation = glm::vec3(property.properties.at(3).value(), - property.properties.at(4).value(), - property.properties.at(5).value()); - - } else if (property.properties.at(0) == "ScalingPivot") { - scalePivot = glm::vec3(property.properties.at(3).value(), - property.properties.at(4).value(), - property.properties.at(5).value()); - - } else if (property.properties.at(0) == "Lcl Scaling") { - scale = glm::vec3(property.properties.at(3).value(), - property.properties.at(4).value(), - property.properties.at(5).value()); - } - } - } + properties = true; + propertyName = "Property"; + index = 3; + } else if (subobject.name == "Properties70") { + properties = true; + propertyName = "P"; + index = 4; + } + if (properties) { foreach (const FBXNode& property, subobject.children) { - if (property.name == "P") { + if (property.name == propertyName) { if (property.properties.at(0) == "Lcl Translation") { - translation = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); + translation = getVec3(property.properties, index); } else if (property.properties.at(0) == "RotationOffset") { - rotationOffset = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); - + rotationOffset = getVec3(property.properties, index); + } else if (property.properties.at(0) == "RotationPivot") { - rotationPivot = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); + rotationPivot = getVec3(property.properties, index); } else if (property.properties.at(0) == "PreRotation") { - preRotation = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); + preRotation = getVec3(property.properties, index); } else if (property.properties.at(0) == "Lcl Rotation") { - rotation = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); + rotation = getVec3(property.properties, index); } else if (property.properties.at(0) == "PostRotation") { - postRotation = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); + postRotation = getVec3(property.properties, index); } else if (property.properties.at(0) == "ScalingPivot") { - scalePivot = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); + scalePivot = getVec3(property.properties, index); } else if (property.properties.at(0) == "Lcl Scaling") { - scale = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); + scale = getVec3(property.properties, index); } } } @@ -840,21 +797,30 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } else if (object.name == "Material") { Material material = { glm::vec3(1.0f, 1.0f, 1.0f), glm::vec3(1.0f, 1.0f, 1.0f), 96.0f }; foreach (const FBXNode& subobject, object.children) { - if (subobject.name == "Properties70") { + bool properties = false; + QByteArray propertyName; + int index; + if (subobject.name == "Properties60") { + properties = true; + propertyName = "Property"; + index = 3; + + } else if (subobject.name == "Properties70") { + properties = true; + propertyName = "P"; + index = 4; + } + if (properties) { foreach (const FBXNode& property, subobject.children) { - if (property.name == "P") { + if (property.name == propertyName) { if (property.properties.at(0) == "DiffuseColor") { - material.diffuse = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); + material.diffuse = getVec3(property.properties, index); } else if (property.properties.at(0) == "SpecularColor") { - material.specular = glm::vec3(property.properties.at(4).value(), - property.properties.at(5).value(), - property.properties.at(6).value()); + material.specular = getVec3(property.properties, index); } else if (property.properties.at(0) == "Shininess") { - material.shininess = property.properties.at(4).value(); + material.shininess = property.properties.at(index).value(); } } } @@ -999,13 +965,21 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::mat4 modelTransform = getGlobalTransform(parentMap, models, modelID); // look for textures, material properties - int partIndex = 0; + int partIndex = mesh.parts.size() - 1; foreach (const QString& childID, childMap.values(modelID)) { - if (!materials.contains(childID) || partIndex >= mesh.parts.size()) { + if (partIndex < 0) { + break; + } + FBXMeshPart& part = mesh.parts[partIndex]; + if (textureFilenames.contains(childID)) { + part.diffuseFilename = textureFilenames.value(childID); + qDebug() << "hello " << part.diffuseFilename << "\n"; + continue; + } + if (!materials.contains(childID)) { continue; } Material material = materials.value(childID); - FBXMeshPart& part = mesh.parts[mesh.parts.size() - ++partIndex]; part.diffuseColor = material.diffuse; part.specularColor = material.specular; part.shininess = material.shininess; @@ -1017,6 +991,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) if (!bumpTextureID.isNull()) { part.normalFilename = textureFilenames.value(bumpTextureID); } + partIndex--; } // find the clusters with which the mesh is associated From 349f7b363a219fb73392a649b817d83272b5b7a2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 21 Oct 2013 16:14:38 -0700 Subject: [PATCH 14/27] More work on custom skeletons. --- interface/src/avatar/FaceModel.cpp | 53 ++++++++++++++-------- interface/src/avatar/FaceModel.h | 7 +-- interface/src/avatar/SkeletonModel.cpp | 61 ++++++++++++++++++++++++++ interface/src/avatar/SkeletonModel.h | 6 +++ interface/src/renderer/FBXReader.cpp | 23 ++++++++-- interface/src/renderer/FBXReader.h | 2 + interface/src/renderer/Model.cpp | 38 +++++++--------- interface/src/renderer/Model.h | 16 +++---- 8 files changed, 149 insertions(+), 57 deletions(-) diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index 974d68562c..bd005353b7 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -6,6 +6,8 @@ // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // +#include + #include "Avatar.h" #include "FaceModel.h" #include "Head.h" @@ -34,22 +36,37 @@ void FaceModel::simulate(float deltaTime) { Model::simulate(deltaTime); } -void FaceModel::maybeUpdateNeckRotation(const FBXJoint& joint, JointState& state) { - // get the rotation axes in joint space and use them to adjust the rotation - glm::mat3 axes = glm::mat3_cast(getRotation()); - glm::mat3 inverse = glm::inverse(glm::mat3(_jointStates[joint.parentIndex].transform * - joint.preRotation * glm::mat4_cast(joint.rotation))); - state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * - glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * - glm::angleAxis(_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; -} - -void FaceModel::maybeUpdateEyeRotation(const FBXJoint& joint, JointState& state) { - // get the lookat position in joint space and use it to adjust the rotation - glm::mat4 inverse = glm::inverse(_jointStates[joint.parentIndex].transform * - joint.preRotation * glm::mat4_cast(joint.rotation)); - glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getOrientation() * IDENTITY_FRONT, 0.0f)); - glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getLookAtPosition() + - _owningHead->getSaccade(), 1.0f)); - state.rotation = rotationBetween(front, lookAt) * joint.rotation; +void FaceModel::updateJointState(int index) { + JointState& state = _jointStates[index]; + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + const FBXJoint& joint = geometry.joints.at(index); + + if (joint.parentIndex == -1) { + glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * + glm::scale(_scale) * glm::translate(_offset); + state.transform = baseTransform * geometry.offset * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + + } else { + if (index == geometry.neckJointIndex) { + // get the rotation axes in joint space and use them to adjust the rotation + glm::mat3 axes = glm::mat3_cast(getRotation()); + glm::mat3 inverse = glm::inverse(glm::mat3(_jointStates[joint.parentIndex].transform * + joint.preRotation * glm::mat4_cast(joint.rotation))); + state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * + glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * + glm::angleAxis(_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; + + } else if (index == geometry.leftEyeJointIndex || index == geometry.rightEyeJointIndex) { + // likewise with the eye joints + glm::mat4 inverse = glm::inverse(_jointStates[joint.parentIndex].transform * + joint.preRotation * glm::mat4_cast(joint.rotation)); + glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getOrientation() * IDENTITY_FRONT, 0.0f)); + glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getLookAtPosition() + + _owningHead->getSaccade(), 1.0f)); + state.rotation = rotationBetween(front, lookAt) * joint.rotation; + } + state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + } } diff --git a/interface/src/avatar/FaceModel.h b/interface/src/avatar/FaceModel.h index 4b9cd66cca..cd0a562298 100644 --- a/interface/src/avatar/FaceModel.h +++ b/interface/src/avatar/FaceModel.h @@ -25,11 +25,8 @@ public: protected: - /// Applies neck rotation based on head orientation. - virtual void maybeUpdateNeckRotation(const FBXJoint& joint, JointState& state); - - /// Applies eye rotation based on lookat position. - virtual void maybeUpdateEyeRotation(const FBXJoint& joint, JointState& state); + /// Updates the state of the joint at the specified index. + virtual void updateJointState(int index); private: diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 5ae193135b..0b2b426ef0 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -6,6 +6,8 @@ // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // +#include + #include "Avatar.h" #include "SkeletonModel.h" @@ -26,3 +28,62 @@ void SkeletonModel::simulate(float deltaTime) { Model::simulate(deltaTime); } + +bool SkeletonModel::render(float alpha) { + if (_jointStates.isEmpty()) { + return false; + } + + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + + for (int i = 0; i < _jointStates.size(); i++) { + const JointState& joint = _jointStates.at(i); + + glPushMatrix(); + glMultMatrixf((const GLfloat*)&joint.transform); + + if (i == geometry.rootJointIndex) { + glColor3f(1.0f, 0.0f, 0.0f); + } else { + glColor3f(1.0f, 1.0f, 1.0f); + } + glutSolidSphere(0.05f, 10, 10); + + glPopMatrix(); + } + + Model::render(alpha); + + return true; +} + +void SkeletonModel::updateJointState(int index) { + JointState& state = _jointStates[index]; + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + const FBXJoint& joint = geometry.joints.at(index); + + if (joint.parentIndex == -1) { + glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * + glm::scale(_scale) * glm::translate(_offset); + state.transform = baseTransform * geometry.offset * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + + } else { + if (index == geometry.leanJointIndex) { + // get the rotation axes in joint space and use them to adjust the rotation + glm::mat3 axes = glm::mat3_cast(_rotation); + glm::mat3 inverse = glm::inverse(glm::mat3(_jointStates[joint.parentIndex].transform * + joint.preRotation * glm::mat4_cast(joint.rotation))); + state.rotation = glm::angleAxis(_owningAvatar->getHead().getLeanSideways(), glm::normalize(inverse * axes[2])) * + glm::angleAxis(_owningAvatar->getHead().getLeanForward(), glm::normalize(inverse * axes[0])) * joint.rotation; + } + state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + } + + if (index == geometry.rootJointIndex) { + state.transform[3][0] = _translation.x; + state.transform[3][1] = _translation.y; + state.transform[3][2] = _translation.z; + } +} diff --git a/interface/src/avatar/SkeletonModel.h b/interface/src/avatar/SkeletonModel.h index ea25058a11..9779a31656 100644 --- a/interface/src/avatar/SkeletonModel.h +++ b/interface/src/avatar/SkeletonModel.h @@ -22,6 +22,12 @@ public: SkeletonModel(Avatar* owningAvatar); void simulate(float deltaTime); + bool render(float alpha); + +protected: + + /// Updates the state of the joint at the specified index. + virtual void updateJointState(int index); private: diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index ef760ddd56..9378bc6ea7 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -660,9 +660,13 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QByteArray jointEyeLeftName = joints.value("jointEyeLeft", "jointEyeLeft").toByteArray(); QByteArray jointEyeRightName = joints.value("jointEyeRight", "jointEyeRight").toByteArray(); QByteArray jointNeckName = joints.value("jointNeck", "jointNeck").toByteArray(); + QByteArray jointRootName = joints.value("jointRoot", "jointRoot").toByteArray(); + QByteArray jointLeanName = joints.value("jointLean", "jointLean").toByteArray(); QString jointEyeLeftID; QString jointEyeRightID; QString jointNeckID; + QString jointRootID; + QString jointLeanID; QVariantHash blendshapeMappings = mapping.value("bs").toHash(); QHash > blendshapeIndices; @@ -711,8 +715,14 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) blendshapes.append(extracted); } } else if (object.name == "Model") { - QByteArray name = object.properties.at(1).toByteArray(); - name = name.left(name.indexOf('\0')); + QByteArray name; + if (object.properties.size() == 3) { + name = object.properties.at(1).toByteArray(); + name = name.left(name.indexOf('\0')); + + } else { + name = object.properties.at(0).toByteArray(); + } if (name == jointEyeLeftName || name == "EyeL" || name == "joint_Leye") { jointEyeLeftID = object.properties.at(0).toString(); @@ -721,6 +731,12 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } else if (name == jointNeckName || name == "NeckRot" || name == "joint_neck") { jointNeckID = object.properties.at(0).toString(); + + } else if (name == jointRootName) { + jointRootID = object.properties.at(0).toString(); + + } else if (name == jointLeanName) { + jointLeanID = object.properties.at(0).toString(); } glm::vec3 translation; glm::vec3 rotationOffset; @@ -947,6 +963,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) geometry.leftEyeJointIndex = modelIDs.indexOf(jointEyeLeftID); geometry.rightEyeJointIndex = modelIDs.indexOf(jointEyeRightID); geometry.neckJointIndex = modelIDs.indexOf(jointNeckID); + geometry.rootJointIndex = modelIDs.indexOf(jointRootID); + geometry.leanJointIndex = modelIDs.indexOf(jointLeanID); // extract the translation component of the neck transform if (geometry.neckJointIndex != -1) { @@ -973,7 +991,6 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) FBXMeshPart& part = mesh.parts[partIndex]; if (textureFilenames.contains(childID)) { part.diffuseFilename = textureFilenames.value(childID); - qDebug() << "hello " << part.diffuseFilename << "\n"; continue; } if (!materials.contains(childID)) { diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 3f86c907f7..9b03f4a475 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -109,6 +109,8 @@ public: int leftEyeJointIndex; int rightEyeJointIndex; int neckJointIndex; + int rootJointIndex; + int leanJointIndex; glm::vec3 neckPivot; }; diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index d236c14c65..15dcf92e87 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -91,22 +91,7 @@ void Model::simulate(float deltaTime) { // update the world space transforms for all joints for (int i = 0; i < _jointStates.size(); i++) { - JointState& state = _jointStates[i]; - const FBXJoint& joint = geometry.joints.at(i); - if (joint.parentIndex == -1) { - state.transform = baseTransform * geometry.offset * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; - - } else { - if (i == geometry.neckJointIndex) { - maybeUpdateNeckRotation(joint, state); - - } else if (i == geometry.leftEyeJointIndex || i == geometry.rightEyeJointIndex) { - maybeUpdateEyeRotation(joint, state); - } - state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; - } + updateJointState(i); } for (int i = 0; i < _meshStates.size(); i++) { @@ -396,12 +381,21 @@ glm::vec4 Model::computeAverageColor() const { return _geometry ? _geometry->computeAverageColor() : glm::vec4(1.0f, 1.0f, 1.0f, 1.0f); } -void Model::maybeUpdateNeckRotation(const FBXJoint& joint, JointState& state) { - // nothing by default -} - -void Model::maybeUpdateEyeRotation(const FBXJoint& joint, JointState& state) { - // nothing by default +void Model::updateJointState(int index) { + JointState& state = _jointStates[index]; + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + const FBXJoint& joint = geometry.joints.at(index); + + if (joint.parentIndex == -1) { + glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * + glm::scale(_scale) * glm::translate(_offset); + state.transform = baseTransform * geometry.offset * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + + } else { + state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + } } void Model::deleteGeometry() { diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index eb145100e2..c84ecac6f3 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -65,6 +65,11 @@ protected: QSharedPointer _geometry; + glm::vec3 _translation; + glm::quat _rotation; + glm::vec3 _scale; + glm::vec3 _offset; + class JointState { public: glm::quat rotation; @@ -72,21 +77,14 @@ protected: }; QVector _jointStates; - - /// Gives subclasses a chance to update the neck joint rotation after its parents and before its children. - virtual void maybeUpdateNeckRotation(const FBXJoint& joint, JointState& state); - /// Gives subclasses a chance to update an eye joint rotation after its parents and before its children. - virtual void maybeUpdateEyeRotation(const FBXJoint& joint, JointState& state); + /// Updates the state of the joint at the specified index. + virtual void updateJointState(int index); private: void deleteGeometry(); - glm::vec3 _translation; - glm::quat _rotation; - glm::vec3 _scale; - glm::vec3 _offset; float _pupilDilation; std::vector _blendshapeCoefficients; From b5f2dcfa550b01ca67146898d4b94f26df92b5be Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 21 Oct 2013 16:58:01 -0700 Subject: [PATCH 15/27] Use the head position from the loaded skeleton, if available. --- interface/src/avatar/Avatar.cpp | 6 +++++- interface/src/avatar/MyAvatar.cpp | 6 +++++- interface/src/renderer/FBXReader.cpp | 6 ++++++ interface/src/renderer/FBXReader.h | 1 + interface/src/renderer/Model.cpp | 26 ++++++++++++++++---------- interface/src/renderer/Model.h | 6 ++++++ 6 files changed, 39 insertions(+), 12 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index a6750f820c..c7acfd149a 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -413,7 +413,11 @@ void Avatar::simulate(float deltaTime, Transmitter* transmitter) { } _head.setBodyRotation(glm::vec3(_bodyPitch, _bodyYaw, _bodyRoll)); - _head.setPosition(_bodyBall[ BODY_BALL_HEAD_BASE ].position); + glm::vec3 headPosition; + if (!_skeletonModel.getHeadPosition(headPosition)) { + headPosition = _bodyBall[BODY_BALL_HEAD_BASE].position; + } + _head.setPosition(headPosition); _head.setSkinColor(glm::vec3(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2])); _head.simulate(deltaTime, false); _skeletonModel.simulate(deltaTime); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 25c202dc45..b7ec4ce4dc 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -318,7 +318,11 @@ void MyAvatar::simulate(float deltaTime, Transmitter* transmitter) { } _head.setBodyRotation(glm::vec3(_bodyPitch, _bodyYaw, _bodyRoll)); - _head.setPosition(_bodyBall[ BODY_BALL_HEAD_BASE ].position); + glm::vec3 headPosition; + if (!_skeletonModel.getHeadPosition(headPosition)) { + headPosition = _bodyBall[BODY_BALL_HEAD_BASE].position; + } + _head.setPosition(headPosition); _head.setScale(_scale); _head.setSkinColor(glm::vec3(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2])); _head.simulate(deltaTime, true); diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 9378bc6ea7..3a2c2607a0 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -662,11 +662,13 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QByteArray jointNeckName = joints.value("jointNeck", "jointNeck").toByteArray(); QByteArray jointRootName = joints.value("jointRoot", "jointRoot").toByteArray(); QByteArray jointLeanName = joints.value("jointLean", "jointLean").toByteArray(); + QByteArray jointHeadName = joints.value("jointHead", "jointHead").toByteArray(); QString jointEyeLeftID; QString jointEyeRightID; QString jointNeckID; QString jointRootID; QString jointLeanID; + QString jointHeadID; QVariantHash blendshapeMappings = mapping.value("bs").toHash(); QHash > blendshapeIndices; @@ -737,6 +739,9 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } else if (name == jointLeanName) { jointLeanID = object.properties.at(0).toString(); + + } else if (name == jointHeadName) { + jointHeadID = object.properties.at(0).toString(); } glm::vec3 translation; glm::vec3 rotationOffset; @@ -965,6 +970,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) geometry.neckJointIndex = modelIDs.indexOf(jointNeckID); geometry.rootJointIndex = modelIDs.indexOf(jointRootID); geometry.leanJointIndex = modelIDs.indexOf(jointLeanID); + geometry.headJointIndex = modelIDs.indexOf(jointHeadID); // extract the translation component of the neck transform if (geometry.neckJointIndex != -1) { diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 9b03f4a475..9f96a67634 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -111,6 +111,7 @@ public: int neckJointIndex; int rootJointIndex; int leanJointIndex; + int headJointIndex; glm::vec3 neckPivot; }; diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 15dcf92e87..8723c6f3b9 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -347,20 +347,17 @@ bool Model::render(float alpha) { return true; } +bool Model::getHeadPosition(glm::vec3& headPosition) const { + return isActive() && getJointPosition(_geometry->getFBXGeometry().headJointIndex, headPosition); +} + bool Model::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const { - if (!isActive() || _jointStates.isEmpty()) { + if (!isActive()) { return false; } const FBXGeometry& geometry = _geometry->getFBXGeometry(); - if (geometry.leftEyeJointIndex != -1) { - const glm::mat4& transform = _jointStates[geometry.leftEyeJointIndex].transform; - firstEyePosition = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); - } - if (geometry.rightEyeJointIndex != -1) { - const glm::mat4& transform = _jointStates[geometry.rightEyeJointIndex].transform; - secondEyePosition = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); - } - return geometry.leftEyeJointIndex != -1 && geometry.rightEyeJointIndex != -1; + return getJointPosition(geometry.leftEyeJointIndex, firstEyePosition) && + getJointPosition(geometry.rightEyeJointIndex, secondEyePosition); } void Model::setURL(const QUrl& url) { @@ -398,6 +395,15 @@ void Model::updateJointState(int index) { } } +bool Model::getJointPosition(int jointIndex, glm::vec3& position) const { + if (jointIndex == -1 || _jointStates.isEmpty()) { + return false; + } + const glm::mat4& transform = _jointStates[jointIndex].transform; + position = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); + return true; +} + void Model::deleteGeometry() { foreach (GLuint id, _blendedVertexBufferIDs) { glDeleteBuffers(1, &id); diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index c84ecac6f3..8ec1df44d2 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -54,6 +54,10 @@ public: Q_INVOKABLE void setURL(const QUrl& url); const QUrl& getURL() const { return _url; } + /// Returns the position of the head joint. + /// \return whether or not the head was found + bool getHeadPosition(glm::vec3& headPosition) const; + /// Retrieve the positions of up to two eye meshes. /// \return whether or not both eye meshes were found bool getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const; @@ -83,6 +87,8 @@ protected: private: + bool getJointPosition(int jointIndex, glm::vec3& position) const; + void deleteGeometry(); float _pupilDilation; From 9d89baa506fe26db68ff4ba1f2a063fe6a6e8d44 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 21 Oct 2013 17:30:26 -0700 Subject: [PATCH 16/27] Apply the neck position. --- interface/src/avatar/FaceModel.cpp | 15 ++++++++++++--- interface/src/renderer/Model.cpp | 21 +++++++++++++++++---- interface/src/renderer/Model.h | 9 +++++++++ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index bd005353b7..494bc5b59f 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -22,9 +22,18 @@ void FaceModel::simulate(float deltaTime) { return; } - const Skeleton& skeleton = static_cast(_owningHead->_owningAvatar)->getSkeleton(); - setTranslation(skeleton.joint[AVATAR_JOINT_NECK_BASE].position); - setRotation(skeleton.joint[AVATAR_JOINT_NECK_BASE].absoluteRotation); + Avatar* owningAvatar = static_cast(_owningHead->_owningAvatar); + const Skeleton& skeleton = owningAvatar->getSkeleton(); + glm::vec3 neckPosition; + if (!owningAvatar->getSkeletonModel().getNeckPosition(neckPosition)) { + neckPosition = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].position; + } + setTranslation(neckPosition); + glm::quat neckRotation; + if (true || !owningAvatar->getSkeletonModel().getNeckRotation(neckRotation)) { + neckRotation = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].absoluteRotation; + } + setRotation(neckRotation); const float MODEL_SCALE = 0.0006f; setScale(glm::vec3(-1.0f, 1.0f, -1.0f) * _owningHead->getScale() * MODEL_SCALE); const glm::vec3 MODEL_TRANSLATION(0.0f, -60.0f, 40.0f); // temporary fudge factor diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 8723c6f3b9..c3aa074f05 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -85,10 +85,6 @@ void Model::simulate(float deltaTime) { _resetStates = true; } - // create our root transform - glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * - glm::scale(_scale) * glm::translate(_offset); - // update the world space transforms for all joints for (int i = 0; i < _jointStates.size(); i++) { updateJointState(i); @@ -351,6 +347,14 @@ bool Model::getHeadPosition(glm::vec3& headPosition) const { return isActive() && getJointPosition(_geometry->getFBXGeometry().headJointIndex, headPosition); } +bool Model::getNeckPosition(glm::vec3& neckPosition) const { + return isActive() && getJointPosition(_geometry->getFBXGeometry().neckJointIndex, neckPosition); +} + +bool Model::getNeckRotation(glm::quat& neckRotation) const { + return isActive() && getJointRotation(_geometry->getFBXGeometry().neckJointIndex, neckRotation); +} + bool Model::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const { if (!isActive()) { return false; @@ -404,6 +408,15 @@ bool Model::getJointPosition(int jointIndex, glm::vec3& position) const { return true; } +bool Model::getJointRotation(int jointIndex, glm::quat& rotation) const { + if (jointIndex == -1 || _jointStates.isEmpty()) { + return false; + } + const glm::mat4& transform = _jointStates[jointIndex].transform; + rotation = glm::normalize(glm::quat_cast(transform)); + return true; +} + void Model::deleteGeometry() { foreach (GLuint id, _blendedVertexBufferIDs) { glDeleteBuffers(1, &id); diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index 8ec1df44d2..4d848057d8 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -58,6 +58,14 @@ public: /// \return whether or not the head was found bool getHeadPosition(glm::vec3& headPosition) const; + /// Returns the position of the neck joint. + /// \return whether or not the neck was found + bool getNeckPosition(glm::vec3& neckPosition) const; + + /// Returns the rotation of the neck joint. + /// \return whether or not the neck was found + bool getNeckRotation(glm::quat& neckRotation) const; + /// Retrieve the positions of up to two eye meshes. /// \return whether or not both eye meshes were found bool getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const; @@ -88,6 +96,7 @@ protected: private: bool getJointPosition(int jointIndex, glm::vec3& position) const; + bool getJointRotation(int jointIndex, glm::quat& rotation) const; void deleteGeometry(); From a33c7ce20ceb5f31ce5397d6e85c673f1f68a2df Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 22 Oct 2013 12:54:32 -0700 Subject: [PATCH 17/27] More work on extracting rotations. --- interface/src/Util.cpp | 60 ++++++++++++++++++++++++++ interface/src/Util.h | 4 ++ interface/src/avatar/Avatar.cpp | 2 +- interface/src/avatar/FaceModel.cpp | 21 +++++---- interface/src/avatar/MyAvatar.cpp | 2 +- interface/src/avatar/SkeletonModel.cpp | 15 ++++--- interface/src/renderer/FBXReader.cpp | 39 +++++++++++------ interface/src/renderer/FBXReader.h | 7 ++- interface/src/renderer/Model.cpp | 20 +++++---- interface/src/renderer/Model.h | 1 + 10 files changed, 132 insertions(+), 39 deletions(-) diff --git a/interface/src/Util.cpp b/interface/src/Util.cpp index 8898ab9eab..2057198daf 100644 --- a/interface/src/Util.cpp +++ b/interface/src/Util.cpp @@ -158,6 +158,66 @@ glm::quat safeMix(const glm::quat& q1, const glm::quat& q2, float proportion) { return glm::normalize(glm::quat(s0 * q1.w + s1 * ow, s0 * q1.x + s1 * ox, s0 * q1.y + s1 * oy, s0 * q1.z + s1 * oz)); } +glm::vec3 extractTranslation(const glm::mat4& matrix) { + return glm::vec3(matrix[3][0], matrix[3][1], matrix[3][2]); +} + +glm::quat extractRotation(const glm::mat4& matrix, bool assumeOrthogonal) { + // uses the iterative polar decomposition algorithm described by Ken Shoemake at + // http://www.cs.wisc.edu/graphics/Courses/838-s2002/Papers/polar-decomp.pdf + // code adapted from Clyde, https://github.com/threerings/clyde/blob/master/src/main/java/com/threerings/math/Matrix4f.java + + // start with the contents of the upper 3x3 portion of the matrix + glm::mat3 upper = glm::mat3(matrix); + if (!assumeOrthogonal) { + for (int i = 0; i < 10; i++) { + // store the results of the previous iteration + glm::mat3 previous = upper; + + // compute average of the matrix with its inverse transpose + float sd00 = previous[1][1] * previous[2][2] - previous[2][1] * previous[1][2]; + float sd10 = previous[0][1] * previous[2][2] - previous[2][1] * previous[0][2]; + float sd20 = previous[0][1] * previous[1][2] - previous[1][1] * previous[0][2]; + float det = previous[0][0] * sd00 + previous[2][0] * sd20 - previous[1][0] * sd10; + if (fabs(det) == 0.0f) { + // determinant is zero; matrix is not invertible + break; + } + float hrdet = 0.5f / det; + upper[0][0] = +sd00 * hrdet + previous[0][0] * 0.5f; + upper[1][0] = -sd10 * hrdet + previous[1][0] * 0.5f; + upper[2][0] = +sd20 * hrdet + previous[2][0] * 0.5f; + + upper[0][1] = -(previous[1][0] * previous[2][2] - previous[2][0] * previous[1][2]) * hrdet + previous[0][1] * 0.5f; + upper[1][1] = +(previous[0][0] * previous[2][2] - previous[2][0] * previous[0][2]) * hrdet + previous[1][1] * 0.5f; + upper[2][1] = -(previous[0][0] * previous[1][2] - previous[1][0] * previous[0][2]) * hrdet + previous[2][1] * 0.5f; + + upper[0][2] = +(previous[1][0] * previous[2][1] - previous[2][0] * previous[1][1]) * hrdet + previous[0][2] * 0.5f; + upper[1][2] = -(previous[0][0] * previous[2][1] - previous[2][0] * previous[0][1]) * hrdet + previous[1][2] * 0.5f; + upper[2][2] = +(previous[0][0] * previous[1][1] - previous[1][0] * previous[0][1]) * hrdet + previous[2][2] * 0.5f; + + // compute the difference; if it's small enough, we're done + glm::mat3 diff = upper - previous; + if (diff[0][0] * diff[0][0] + diff[1][0] * diff[1][0] + diff[2][0] * diff[2][0] + diff[0][1] * diff[0][1] + + diff[1][1] * diff[1][1] + diff[2][1] * diff[2][1] + diff[0][2] * diff[0][2] + diff[1][2] * diff[1][2] + + diff[2][2] * diff[2][2] < EPSILON) { + break; + } + } + } + + // now that we have a nice orthogonal matrix, we can extract the rotation quaternion + // using the method described in http://en.wikipedia.org/wiki/Rotation_matrix#Conversions + float x2 = fabs(1.0f + upper[0][0] - upper[1][1] - upper[2][2]); + float y2 = fabs(1.0f - upper[0][0] + upper[1][1] - upper[2][2]); + float z2 = fabs(1.0f - upper[0][0] - upper[1][1] + upper[2][2]); + float w2 = fabs(1.0f + upper[0][0] + upper[1][1] + upper[2][2]); + return glm::normalize(glm::quat(0.5f * sqrtf(w2), + 0.5f * sqrtf(x2) * (upper[1][2] >= upper[2][1] ? 1.0f : -1.0f), + 0.5f * sqrtf(y2) * (upper[2][0] >= upper[0][2] ? 1.0f : -1.0f), + 0.5f * sqrtf(z2) * (upper[0][1] >= upper[1][0] ? 1.0f : -1.0f))); +} + // Draw a 3D vector floating in space void drawVector(glm::vec3 * vector) { glDisable(GL_LIGHTING); diff --git a/interface/src/Util.h b/interface/src/Util.h index 3039ab1a31..f304535d70 100644 --- a/interface/src/Util.h +++ b/interface/src/Util.h @@ -55,6 +55,10 @@ glm::vec3 safeEulerAngles(const glm::quat& q); glm::quat safeMix(const glm::quat& q1, const glm::quat& q2, float alpha); +glm::vec3 extractTranslation(const glm::mat4& matrix); + +glm::quat extractRotation(const glm::mat4& matrix, bool assumeOrthogonal = false); + double diffclock(timeval *clock1,timeval *clock2); void renderGroundPlaneGrid(float size, float impact); diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index c7acfd149a..272604e58c 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -412,6 +412,7 @@ void Avatar::simulate(float deltaTime, Transmitter* transmitter) { } } + _skeletonModel.simulate(deltaTime); _head.setBodyRotation(glm::vec3(_bodyPitch, _bodyYaw, _bodyRoll)); glm::vec3 headPosition; if (!_skeletonModel.getHeadPosition(headPosition)) { @@ -420,7 +421,6 @@ void Avatar::simulate(float deltaTime, Transmitter* transmitter) { _head.setPosition(headPosition); _head.setSkinColor(glm::vec3(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2])); _head.simulate(deltaTime, false); - _skeletonModel.simulate(deltaTime); _hand.simulate(deltaTime, false); // use speed and angular velocity to determine walking vs. standing diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index 494bc5b59f..eb2ca70805 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -30,7 +30,7 @@ void FaceModel::simulate(float deltaTime) { } setTranslation(neckPosition); glm::quat neckRotation; - if (true || !owningAvatar->getSkeletonModel().getNeckRotation(neckRotation)) { + if (!owningAvatar->getSkeletonModel().getNeckRotation(neckRotation)) { neckRotation = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].absoluteRotation; } setRotation(neckRotation); @@ -50,18 +50,21 @@ void FaceModel::updateJointState(int index) { const FBXGeometry& geometry = _geometry->getFBXGeometry(); const FBXJoint& joint = geometry.joints.at(index); + glm::quat combinedRotation = joint.preRotation * state.rotation * joint.postRotation; if (joint.parentIndex == -1) { glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * glm::scale(_scale) * glm::translate(_offset); - state.transform = baseTransform * geometry.offset * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; + + state.transform = baseTransform * geometry.offset * joint.preTransform * + glm::mat4_cast(combinedRotation) * joint.postTransform; + state.combinedRotation = _rotation * combinedRotation; } else { if (index == geometry.neckJointIndex) { // get the rotation axes in joint space and use them to adjust the rotation glm::mat3 axes = glm::mat3_cast(getRotation()); glm::mat3 inverse = glm::inverse(glm::mat3(_jointStates[joint.parentIndex].transform * - joint.preRotation * glm::mat4_cast(joint.rotation))); + joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation))); state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * glm::angleAxis(_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; @@ -69,13 +72,15 @@ void FaceModel::updateJointState(int index) { } else if (index == geometry.leftEyeJointIndex || index == geometry.rightEyeJointIndex) { // likewise with the eye joints glm::mat4 inverse = glm::inverse(_jointStates[joint.parentIndex].transform * - joint.preRotation * glm::mat4_cast(joint.rotation)); + joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation)); glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getOrientation() * IDENTITY_FRONT, 0.0f)); glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getLookAtPosition() + _owningHead->getSaccade(), 1.0f)); state.rotation = rotationBetween(front, lookAt) * joint.rotation; - } - state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; + } + const JointState& parentState = _jointStates.at(joint.parentIndex); + state.transform = parentState.transform * joint.preTransform * + glm::mat4_cast(combinedRotation) * joint.postTransform; + state.combinedRotation = parentState.combinedRotation * combinedRotation; } } diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index b7ec4ce4dc..b32af7dba6 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -317,6 +317,7 @@ void MyAvatar::simulate(float deltaTime, Transmitter* transmitter) { } } + _skeletonModel.simulate(deltaTime); _head.setBodyRotation(glm::vec3(_bodyPitch, _bodyYaw, _bodyRoll)); glm::vec3 headPosition; if (!_skeletonModel.getHeadPosition(headPosition)) { @@ -326,7 +327,6 @@ void MyAvatar::simulate(float deltaTime, Transmitter* transmitter) { _head.setScale(_scale); _head.setSkinColor(glm::vec3(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2])); _head.simulate(deltaTime, true); - _skeletonModel.simulate(deltaTime); _hand.simulate(deltaTime, true); const float WALKING_SPEED_THRESHOLD = 0.2f; diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 0b2b426ef0..fbffa49fc1 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -62,23 +62,28 @@ void SkeletonModel::updateJointState(int index) { const FBXGeometry& geometry = _geometry->getFBXGeometry(); const FBXJoint& joint = geometry.joints.at(index); + glm::quat combinedRotation = joint.preRotation * state.rotation * joint.postRotation; if (joint.parentIndex == -1) { glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * glm::scale(_scale) * glm::translate(_offset); - state.transform = baseTransform * geometry.offset * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; + + state.transform = baseTransform * geometry.offset * joint.preTransform * + glm::mat4_cast(combinedRotation) * joint.postTransform; + state.combinedRotation = _rotation * combinedRotation; } else { if (index == geometry.leanJointIndex) { // get the rotation axes in joint space and use them to adjust the rotation glm::mat3 axes = glm::mat3_cast(_rotation); glm::mat3 inverse = glm::inverse(glm::mat3(_jointStates[joint.parentIndex].transform * - joint.preRotation * glm::mat4_cast(joint.rotation))); + joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation))); state.rotation = glm::angleAxis(_owningAvatar->getHead().getLeanSideways(), glm::normalize(inverse * axes[2])) * glm::angleAxis(_owningAvatar->getHead().getLeanForward(), glm::normalize(inverse * axes[0])) * joint.rotation; } - state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; + const JointState& parentState = _jointStates.at(joint.parentIndex); + state.transform = parentState.transform * joint.preTransform * + glm::mat4_cast(combinedRotation) * joint.postTransform; + state.combinedRotation = parentState.combinedRotation * combinedRotation; } if (index == geometry.rootJointIndex) { diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 3a2c2607a0..143fb2390e 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -17,6 +17,7 @@ #include #include "FBXReader.h" +#include "Util.h" using namespace std; @@ -453,9 +454,11 @@ public: int parentIndex; - glm::mat4 preRotation; + glm::mat4 preTransform; + glm::quat preRotation; glm::quat rotation; - glm::mat4 postRotation; + glm::quat postRotation; + glm::mat4 postTransform; }; glm::mat4 getGlobalTransform(const QMultiHash& parentMap, @@ -463,7 +466,8 @@ glm::mat4 getGlobalTransform(const QMultiHash& parentMap, glm::mat4 globalTransform; while (!nodeID.isNull()) { const FBXModel& model = models.value(nodeID); - globalTransform = model.preRotation * glm::mat4_cast(model.rotation) * model.postRotation * globalTransform; + globalTransform = model.preTransform * glm::mat4_cast(model.preRotation * model.rotation * model.postRotation) * + model.postTransform * globalTransform; QList parentIDs = parentMap.values(nodeID); nodeID = QString(); @@ -798,11 +802,12 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } } // see FBX documentation, http://download.autodesk.com/us/fbx/20112/FBX_SDK_HELP/index.html - model.preRotation = glm::translate(translation) * glm::translate(rotationOffset) * - glm::translate(rotationPivot) * glm::mat4_cast(glm::quat(glm::radians(preRotation))); + model.preTransform = glm::translate(translation) * glm::translate(rotationOffset) * + glm::translate(rotationPivot); + model.preRotation = glm::quat(glm::radians(preRotation)); model.rotation = glm::quat(glm::radians(rotation)); - model.postRotation = glm::mat4_cast(glm::quat(glm::radians(postRotation))) * - glm::translate(-rotationPivot) * glm::translate(scalePivot) * + model.postRotation = glm::quat(glm::radians(postRotation)); + model.postTransform = glm::translate(-rotationPivot) * glm::translate(scalePivot) * glm::scale(scale) * glm::translate(-scalePivot); models.insert(object.properties.at(0).toString(), model); @@ -921,10 +926,10 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) // get offset transform from mapping FBXGeometry geometry; float offsetScale = mapping.value("scale", 1.0f).toFloat(); + glm::quat offsetRotation = glm::quat(glm::radians(glm::vec3(mapping.value("rx").toFloat(), + mapping.value("ry").toFloat(), mapping.value("rz").toFloat()))); geometry.offset = glm::translate(mapping.value("tx").toFloat(), mapping.value("ty").toFloat(), - mapping.value("tz").toFloat()) * glm::mat4_cast(glm::quat(glm::radians(glm::vec3(mapping.value("rx").toFloat(), - mapping.value("ry").toFloat(), mapping.value("rz").toFloat())))) * - glm::scale(offsetScale, offsetScale, offsetScale); + mapping.value("tz").toFloat()) * glm::mat4_cast(offsetRotation) * glm::scale(offsetScale, offsetScale, offsetScale); // get the list of models in depth-first traversal order QVector modelIDs; @@ -950,15 +955,21 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) const FBXModel& model = models[modelID]; FBXJoint joint; joint.parentIndex = model.parentIndex; + joint.preTransform = model.preTransform; joint.preRotation = model.preRotation; joint.rotation = model.rotation; joint.postRotation = model.postRotation; - if (joint.parentIndex == -1) { - joint.transform = geometry.offset * model.preRotation * glm::mat4_cast(model.rotation) * model.postRotation; + joint.postTransform = model.postTransform; + glm::quat combinedRotation = model.preRotation * model.rotation * model.postRotation; + if (joint.parentIndex == -1) { + joint.transform = geometry.offset * model.preTransform * glm::mat4_cast(combinedRotation) * model.postTransform; + joint.inverseBindRotation = glm::inverse(offsetRotation * combinedRotation); } else { - joint.transform = geometry.joints.at(joint.parentIndex).transform * - model.preRotation * glm::mat4_cast(model.rotation) * model.postRotation; + const FBXJoint& parentJoint = geometry.joints.at(joint.parentIndex); + joint.transform = parentJoint.transform * + model.preTransform * glm::mat4_cast(combinedRotation) * model.postTransform; + joint.inverseBindRotation = glm::inverse(combinedRotation) * parentJoint.inverseBindRotation; } geometry.joints.append(joint); geometry.jointIndices.insert(model.name, geometry.joints.size() - 1); diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 9f96a67634..17f743ec64 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -43,10 +43,13 @@ class FBXJoint { public: int parentIndex; - glm::mat4 preRotation; + glm::mat4 preTransform; + glm::quat preRotation; glm::quat rotation; - glm::mat4 postRotation; + glm::quat postRotation; + glm::mat4 postTransform; glm::mat4 transform; + glm::quat inverseBindRotation; }; /// A single binding to a joint in an FBX document. diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index c3aa074f05..4f889da997 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -387,15 +387,20 @@ void Model::updateJointState(int index) { const FBXGeometry& geometry = _geometry->getFBXGeometry(); const FBXJoint& joint = geometry.joints.at(index); + glm::quat combinedRotation = joint.preRotation * state.rotation * joint.postRotation; if (joint.parentIndex == -1) { glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * glm::scale(_scale) * glm::translate(_offset); - state.transform = baseTransform * geometry.offset * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; + + state.transform = baseTransform * geometry.offset * joint.preTransform * + glm::mat4_cast(combinedRotation) * joint.postTransform; + state.combinedRotation = _rotation * combinedRotation; } else { - state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation * - glm::mat4_cast(state.rotation) * joint.postRotation; + const JointState& parentState = _jointStates.at(joint.parentIndex); + state.transform = parentState.transform * joint.preTransform * + glm::mat4_cast(combinedRotation) * joint.postTransform; + state.combinedRotation = parentState.combinedRotation * combinedRotation; } } @@ -403,8 +408,7 @@ bool Model::getJointPosition(int jointIndex, glm::vec3& position) const { if (jointIndex == -1 || _jointStates.isEmpty()) { return false; } - const glm::mat4& transform = _jointStates[jointIndex].transform; - position = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); + position = extractTranslation(_jointStates[jointIndex].transform); return true; } @@ -412,8 +416,8 @@ bool Model::getJointRotation(int jointIndex, glm::quat& rotation) const { if (jointIndex == -1 || _jointStates.isEmpty()) { return false; } - const glm::mat4& transform = _jointStates[jointIndex].transform; - rotation = glm::normalize(glm::quat_cast(transform)); + rotation = _jointStates[jointIndex].combinedRotation * + _geometry->getFBXGeometry().joints[jointIndex].inverseBindRotation; return true; } diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index 4d848057d8..894de7bfe1 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -86,6 +86,7 @@ protected: public: glm::quat rotation; glm::mat4 transform; + glm::quat combinedRotation; }; QVector _jointStates; From cd4b88acaa0edf87d612f24fc5ea8b2c93bdd1c1 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 22 Oct 2013 13:39:51 -0700 Subject: [PATCH 18/27] More rotation tweaks. --- interface/src/avatar/FaceModel.cpp | 6 +++--- interface/src/avatar/SkeletonModel.cpp | 4 ++-- interface/src/renderer/FBXReader.cpp | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index eb2ca70805..40f85f62c2 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -23,7 +23,6 @@ void FaceModel::simulate(float deltaTime) { } Avatar* owningAvatar = static_cast(_owningHead->_owningAvatar); - const Skeleton& skeleton = owningAvatar->getSkeleton(); glm::vec3 neckPosition; if (!owningAvatar->getSkeletonModel().getNeckPosition(neckPosition)) { neckPosition = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].position; @@ -31,11 +30,12 @@ void FaceModel::simulate(float deltaTime) { setTranslation(neckPosition); glm::quat neckRotation; if (!owningAvatar->getSkeletonModel().getNeckRotation(neckRotation)) { - neckRotation = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].absoluteRotation; + neckRotation = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].absoluteRotation * + glm::angleAxis(180.0f, 0.0f, 1.0f, 0.0f); } setRotation(neckRotation); const float MODEL_SCALE = 0.0006f; - setScale(glm::vec3(-1.0f, 1.0f, -1.0f) * _owningHead->getScale() * MODEL_SCALE); + setScale(glm::vec3(1.0f, 1.0f, 1.0f) * _owningHead->getScale() * MODEL_SCALE); const glm::vec3 MODEL_TRANSLATION(0.0f, -60.0f, 40.0f); // temporary fudge factor setOffset(MODEL_TRANSLATION - _geometry->getFBXGeometry().neckPivot); diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index fbffa49fc1..fb4c073810 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -22,9 +22,9 @@ void SkeletonModel::simulate(float deltaTime) { } setTranslation(_owningAvatar->getPosition()); - setRotation(_owningAvatar->getOrientation()); + setRotation(_owningAvatar->getOrientation() * glm::angleAxis(180.0f, 0.0f, 1.0f, 0.0f)); const float MODEL_SCALE = 0.0006f; - setScale(glm::vec3(-1.0f, 1.0f, -1.0f) * _owningAvatar->getScale() * MODEL_SCALE); + setScale(glm::vec3(1.0f, 1.0f, 1.0f) * _owningAvatar->getScale() * MODEL_SCALE); Model::simulate(deltaTime); } diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 143fb2390e..8a6abb4844 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include "FBXReader.h" @@ -963,7 +964,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::quat combinedRotation = model.preRotation * model.rotation * model.postRotation; if (joint.parentIndex == -1) { joint.transform = geometry.offset * model.preTransform * glm::mat4_cast(combinedRotation) * model.postTransform; - joint.inverseBindRotation = glm::inverse(offsetRotation * combinedRotation); + joint.inverseBindRotation = glm::inverse(combinedRotation); } else { const FBXJoint& parentJoint = geometry.joints.at(joint.parentIndex); From 02a21ae9c0b6fc8093397c8ddc1340789ad61479 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 22 Oct 2013 14:33:27 -0700 Subject: [PATCH 19/27] New plan: let's get the skeleton working first, then work on the face. --- interface/src/avatar/FaceModel.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index 40f85f62c2..2e7a757f21 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -30,8 +30,7 @@ void FaceModel::simulate(float deltaTime) { setTranslation(neckPosition); glm::quat neckRotation; if (!owningAvatar->getSkeletonModel().getNeckRotation(neckRotation)) { - neckRotation = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].absoluteRotation * - glm::angleAxis(180.0f, 0.0f, 1.0f, 0.0f); + neckRotation = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].absoluteRotation; } setRotation(neckRotation); const float MODEL_SCALE = 0.0006f; From d393bdcea65ef2b65b11b8451970d0db76d4bb80 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 22 Oct 2013 15:04:49 -0700 Subject: [PATCH 20/27] Wheeee, let's try a different way of applying the lean rotation. --- interface/src/avatar/FaceModel.cpp | 6 +++--- interface/src/avatar/SkeletonModel.cpp | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index 2e7a757f21..22686998d3 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -59,10 +59,11 @@ void FaceModel::updateJointState(int index) { state.combinedRotation = _rotation * combinedRotation; } else { + const JointState& parentState = _jointStates.at(joint.parentIndex); if (index == geometry.neckJointIndex) { // get the rotation axes in joint space and use them to adjust the rotation glm::mat3 axes = glm::mat3_cast(getRotation()); - glm::mat3 inverse = glm::inverse(glm::mat3(_jointStates[joint.parentIndex].transform * + glm::mat3 inverse = glm::inverse(glm::mat3(parentState.transform * joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation))); state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * @@ -70,14 +71,13 @@ void FaceModel::updateJointState(int index) { } else if (index == geometry.leftEyeJointIndex || index == geometry.rightEyeJointIndex) { // likewise with the eye joints - glm::mat4 inverse = glm::inverse(_jointStates[joint.parentIndex].transform * + glm::mat4 inverse = glm::inverse(parentState.transform * joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation)); glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getOrientation() * IDENTITY_FRONT, 0.0f)); glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getLookAtPosition() + _owningHead->getSaccade(), 1.0f)); state.rotation = rotationBetween(front, lookAt) * joint.rotation; } - const JointState& parentState = _jointStates.at(joint.parentIndex); state.transform = parentState.transform * joint.preTransform * glm::mat4_cast(combinedRotation) * joint.postTransform; state.combinedRotation = parentState.combinedRotation * combinedRotation; diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index fb4c073810..0688290928 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -72,15 +72,16 @@ void SkeletonModel::updateJointState(int index) { state.combinedRotation = _rotation * combinedRotation; } else { + const JointState& parentState = _jointStates.at(joint.parentIndex); if (index == geometry.leanJointIndex) { // get the rotation axes in joint space and use them to adjust the rotation - glm::mat3 axes = glm::mat3_cast(_rotation); - glm::mat3 inverse = glm::inverse(glm::mat3(_jointStates[joint.parentIndex].transform * - joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation))); - state.rotation = glm::angleAxis(_owningAvatar->getHead().getLeanSideways(), glm::normalize(inverse * axes[2])) * - glm::angleAxis(_owningAvatar->getHead().getLeanForward(), glm::normalize(inverse * axes[0])) * joint.rotation; + state.combinedRotation = _rotation * glm::quat(glm::radians(glm::vec3(_owningAvatar->getHead().getLeanForward(), + 0.0f, _owningAvatar->getHead().getLeanSideways()))) * glm::inverse(_rotation) * parentState.combinedRotation * + joint.preRotation * joint.rotation; + state.rotation = glm::inverse(joint.postRotation * glm::inverse(state.combinedRotation) * + parentState.combinedRotation * joint.preRotation); + combinedRotation = joint.preRotation * state.rotation * joint.postRotation; } - const JointState& parentState = _jointStates.at(joint.parentIndex); state.transform = parentState.transform * joint.preTransform * glm::mat4_cast(combinedRotation) * joint.postTransform; state.combinedRotation = parentState.combinedRotation * combinedRotation; From 1f8bed9d9c392c9f2c60d80bc6d32afe805a0b3e Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 22 Oct 2013 15:31:27 -0700 Subject: [PATCH 21/27] More transform rejiggery. --- interface/src/avatar/FaceModel.cpp | 55 ++++++++------------------ interface/src/avatar/FaceModel.h | 4 +- interface/src/avatar/SkeletonModel.cpp | 41 ++++++------------- interface/src/avatar/SkeletonModel.h | 2 + interface/src/renderer/Model.cpp | 26 +++++++++++- interface/src/renderer/Model.h | 4 ++ 6 files changed, 61 insertions(+), 71 deletions(-) diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index 22686998d3..94b81fcb60 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -44,42 +44,21 @@ void FaceModel::simulate(float deltaTime) { Model::simulate(deltaTime); } -void FaceModel::updateJointState(int index) { - JointState& state = _jointStates[index]; - const FBXGeometry& geometry = _geometry->getFBXGeometry(); - const FBXJoint& joint = geometry.joints.at(index); - - glm::quat combinedRotation = joint.preRotation * state.rotation * joint.postRotation; - if (joint.parentIndex == -1) { - glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * - glm::scale(_scale) * glm::translate(_offset); - - state.transform = baseTransform * geometry.offset * joint.preTransform * - glm::mat4_cast(combinedRotation) * joint.postTransform; - state.combinedRotation = _rotation * combinedRotation; - - } else { - const JointState& parentState = _jointStates.at(joint.parentIndex); - if (index == geometry.neckJointIndex) { - // get the rotation axes in joint space and use them to adjust the rotation - glm::mat3 axes = glm::mat3_cast(getRotation()); - glm::mat3 inverse = glm::inverse(glm::mat3(parentState.transform * - joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation))); - state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * - glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * - glm::angleAxis(_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; - - } else if (index == geometry.leftEyeJointIndex || index == geometry.rightEyeJointIndex) { - // likewise with the eye joints - glm::mat4 inverse = glm::inverse(parentState.transform * - joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation)); - glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getOrientation() * IDENTITY_FRONT, 0.0f)); - glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getLookAtPosition() + - _owningHead->getSaccade(), 1.0f)); - state.rotation = rotationBetween(front, lookAt) * joint.rotation; - } - state.transform = parentState.transform * joint.preTransform * - glm::mat4_cast(combinedRotation) * joint.postTransform; - state.combinedRotation = parentState.combinedRotation * combinedRotation; - } +void FaceModel::maybeUpdateNeckRotation(const JointState& parentState, const FBXJoint& joint, JointState& state) { + // get the rotation axes in joint space and use them to adjust the rotation + glm::mat3 axes = glm::mat3_cast(_rotation); + glm::quat inverse = parentState.combinedRotation * joint.preRotation * joint.rotation; + state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * + glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * + glm::angleAxis(_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; +} + +void FaceModel::maybeUpdateEyeRotation(const JointState& parentState, const FBXJoint& joint, JointState& state) { + // likewise with the eye joints + glm::mat4 inverse = glm::inverse(parentState.transform * + joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation)); + glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getOrientation() * IDENTITY_FRONT, 0.0f)); + glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getLookAtPosition() + + _owningHead->getSaccade(), 1.0f)); + state.rotation = rotationBetween(front, lookAt) * joint.rotation; } diff --git a/interface/src/avatar/FaceModel.h b/interface/src/avatar/FaceModel.h index cd0a562298..acf2d2baf4 100644 --- a/interface/src/avatar/FaceModel.h +++ b/interface/src/avatar/FaceModel.h @@ -25,8 +25,8 @@ public: protected: - /// Updates the state of the joint at the specified index. - virtual void updateJointState(int index); + virtual void maybeUpdateNeckRotation(const JointState& parentState, const FBXJoint& joint, JointState& state); + virtual void maybeUpdateEyeRotation(const JointState& parentState, const FBXJoint& joint, JointState& state); private: diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 0688290928..1a85c46cf6 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -58,38 +58,21 @@ bool SkeletonModel::render(float alpha) { } void SkeletonModel::updateJointState(int index) { - JointState& state = _jointStates[index]; - const FBXGeometry& geometry = _geometry->getFBXGeometry(); - const FBXJoint& joint = geometry.joints.at(index); + Model::updateJointState(index); - glm::quat combinedRotation = joint.preRotation * state.rotation * joint.postRotation; - if (joint.parentIndex == -1) { - glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * - glm::scale(_scale) * glm::translate(_offset); - - state.transform = baseTransform * geometry.offset * joint.preTransform * - glm::mat4_cast(combinedRotation) * joint.postTransform; - state.combinedRotation = _rotation * combinedRotation; - - } else { - const JointState& parentState = _jointStates.at(joint.parentIndex); - if (index == geometry.leanJointIndex) { - // get the rotation axes in joint space and use them to adjust the rotation - state.combinedRotation = _rotation * glm::quat(glm::radians(glm::vec3(_owningAvatar->getHead().getLeanForward(), - 0.0f, _owningAvatar->getHead().getLeanSideways()))) * glm::inverse(_rotation) * parentState.combinedRotation * - joint.preRotation * joint.rotation; - state.rotation = glm::inverse(joint.postRotation * glm::inverse(state.combinedRotation) * - parentState.combinedRotation * joint.preRotation); - combinedRotation = joint.preRotation * state.rotation * joint.postRotation; - } - state.transform = parentState.transform * joint.preTransform * - glm::mat4_cast(combinedRotation) * joint.postTransform; - state.combinedRotation = parentState.combinedRotation * combinedRotation; - } - - if (index == geometry.rootJointIndex) { + if (index == _geometry->getFBXGeometry().rootJointIndex) { + JointState& state = _jointStates[index]; state.transform[3][0] = _translation.x; state.transform[3][1] = _translation.y; state.transform[3][2] = _translation.z; } } + +void SkeletonModel::maybeUpdateLeanRotation(const JointState& parentState, const FBXJoint& joint, JointState& state) { + // get the rotation axes in joint space and use them to adjust the rotation + glm::mat3 axes = glm::mat3_cast(_rotation); + glm::quat inverse = parentState.combinedRotation * joint.preRotation * joint.rotation; + state.rotation = glm::angleAxis(-_owningAvatar->getHead().getLeanSideways(), glm::normalize(inverse * axes[2])) * + glm::angleAxis(-_owningAvatar->getHead().getLeanForward(), glm::normalize(inverse * axes[0])) * joint.rotation; +} + diff --git a/interface/src/avatar/SkeletonModel.h b/interface/src/avatar/SkeletonModel.h index 9779a31656..dfda39d066 100644 --- a/interface/src/avatar/SkeletonModel.h +++ b/interface/src/avatar/SkeletonModel.h @@ -29,6 +29,8 @@ protected: /// Updates the state of the joint at the specified index. virtual void updateJointState(int index); + virtual void maybeUpdateLeanRotation(const JointState& parentState, const FBXJoint& joint, JointState& state); + private: Avatar* _owningAvatar; diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 4f889da997..525f73e196 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -387,23 +387,45 @@ void Model::updateJointState(int index) { const FBXGeometry& geometry = _geometry->getFBXGeometry(); const FBXJoint& joint = geometry.joints.at(index); - glm::quat combinedRotation = joint.preRotation * state.rotation * joint.postRotation; if (joint.parentIndex == -1) { glm::mat4 baseTransform = glm::translate(_translation) * glm::mat4_cast(_rotation) * glm::scale(_scale) * glm::translate(_offset); - + + glm::quat combinedRotation = joint.preRotation * state.rotation * joint.postRotation; state.transform = baseTransform * geometry.offset * joint.preTransform * glm::mat4_cast(combinedRotation) * joint.postTransform; state.combinedRotation = _rotation * combinedRotation; } else { const JointState& parentState = _jointStates.at(joint.parentIndex); + if (index == geometry.leanJointIndex) { + maybeUpdateLeanRotation(parentState, joint, state); + + } else if (index == geometry.neckJointIndex) { + maybeUpdateNeckRotation(parentState, joint, state); + + } else if (index == geometry.leftEyeJointIndex || index == geometry.rightEyeJointIndex) { + maybeUpdateEyeRotation(parentState, joint, state); + } + glm::quat combinedRotation = joint.preRotation * state.rotation * joint.postRotation; state.transform = parentState.transform * joint.preTransform * glm::mat4_cast(combinedRotation) * joint.postTransform; state.combinedRotation = parentState.combinedRotation * combinedRotation; } } +void Model::maybeUpdateLeanRotation(const JointState& parentState, const FBXJoint& joint, JointState& state) { + // nothing by default +} + +void Model::maybeUpdateNeckRotation(const JointState& parentState, const FBXJoint& joint, JointState& state) { + // nothing by default +} + +void Model::maybeUpdateEyeRotation(const JointState& parentState, const FBXJoint& joint, JointState& state) { + // nothing by default +} + bool Model::getJointPosition(int jointIndex, glm::vec3& position) const { if (jointIndex == -1 || _jointStates.isEmpty()) { return false; diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index 894de7bfe1..3bd1b2fa55 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -94,6 +94,10 @@ protected: /// Updates the state of the joint at the specified index. virtual void updateJointState(int index); + virtual void maybeUpdateLeanRotation(const JointState& parentState, const FBXJoint& joint, JointState& state); + virtual void maybeUpdateNeckRotation(const JointState& parentState, const FBXJoint& joint, JointState& state); + virtual void maybeUpdateEyeRotation(const JointState& parentState, const FBXJoint& joint, JointState& state); + private: bool getJointPosition(int jointIndex, glm::vec3& position) const; From 41775912d86d4d6314df43c2033ad71afa51e053 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 22 Oct 2013 15:39:45 -0700 Subject: [PATCH 22/27] Back to using matrices. --- interface/src/avatar/FaceModel.cpp | 3 ++- interface/src/avatar/SkeletonModel.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index 94b81fcb60..3e656ad7f4 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -47,7 +47,8 @@ void FaceModel::simulate(float deltaTime) { void FaceModel::maybeUpdateNeckRotation(const JointState& parentState, const FBXJoint& joint, JointState& state) { // get the rotation axes in joint space and use them to adjust the rotation glm::mat3 axes = glm::mat3_cast(_rotation); - glm::quat inverse = parentState.combinedRotation * joint.preRotation * joint.rotation; + glm::mat3 inverse = glm::mat3(glm::inverse(parentState.transform * + joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation))); state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * glm::angleAxis(_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 1a85c46cf6..7ea1572089 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -71,7 +71,8 @@ void SkeletonModel::updateJointState(int index) { void SkeletonModel::maybeUpdateLeanRotation(const JointState& parentState, const FBXJoint& joint, JointState& state) { // get the rotation axes in joint space and use them to adjust the rotation glm::mat3 axes = glm::mat3_cast(_rotation); - glm::quat inverse = parentState.combinedRotation * joint.preRotation * joint.rotation; + glm::mat3 inverse = glm::mat3(glm::inverse(parentState.transform * + joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation))); state.rotation = glm::angleAxis(-_owningAvatar->getHead().getLeanSideways(), glm::normalize(inverse * axes[2])) * glm::angleAxis(-_owningAvatar->getHead().getLeanForward(), glm::normalize(inverse * axes[0])) * joint.rotation; } From 2e06965b1c62d12c6223978c688fee2958aa931a Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 22 Oct 2013 15:50:42 -0700 Subject: [PATCH 23/27] I think this does the trick, finally. --- interface/src/avatar/FaceModel.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index 3e656ad7f4..dcc7b82f9d 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -30,7 +30,8 @@ void FaceModel::simulate(float deltaTime) { setTranslation(neckPosition); glm::quat neckRotation; if (!owningAvatar->getSkeletonModel().getNeckRotation(neckRotation)) { - neckRotation = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].absoluteRotation; + neckRotation = owningAvatar->getSkeleton().joint[AVATAR_JOINT_NECK_BASE].absoluteRotation * + glm::angleAxis(180.0f, 0.0f, 1.0f, 0.0f); } setRotation(neckRotation); const float MODEL_SCALE = 0.0006f; @@ -49,9 +50,9 @@ void FaceModel::maybeUpdateNeckRotation(const JointState& parentState, const FBX glm::mat3 axes = glm::mat3_cast(_rotation); glm::mat3 inverse = glm::mat3(glm::inverse(parentState.transform * joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation))); - state.rotation = glm::angleAxis(_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * + state.rotation = glm::angleAxis(-_owningHead->getRoll(), glm::normalize(inverse * axes[2])) * glm::angleAxis(_owningHead->getYaw(), glm::normalize(inverse * axes[1])) * - glm::angleAxis(_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; + glm::angleAxis(-_owningHead->getPitch(), glm::normalize(inverse * axes[0])) * joint.rotation; } void FaceModel::maybeUpdateEyeRotation(const JointState& parentState, const FBXJoint& joint, JointState& state) { From 9dd41f42060292137c3cf4622e5c92eca71084c1 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 22 Oct 2013 16:52:44 -0700 Subject: [PATCH 24/27] Render sticks, use avatar colors. --- interface/src/avatar/Avatar.cpp | 19 +++++++++----- interface/src/avatar/Avatar.h | 2 ++ interface/src/avatar/MyAvatar.cpp | 9 ++----- interface/src/avatar/SkeletonModel.cpp | 36 +++++++++++++++++++------- interface/src/renderer/Model.h | 4 +-- 5 files changed, 45 insertions(+), 25 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 272604e58c..ed8ed2a103 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -757,13 +757,8 @@ void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { } } else if (renderAvatarBalls || !(_voxels.getVoxelURL().isValid() || _skeletonModel.isActive())) { // Render the body as balls and cones - glm::vec3 skinColor(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2]); - glm::vec3 darkSkinColor(DARK_SKIN_COLOR[0], DARK_SKIN_COLOR[1], DARK_SKIN_COLOR[2]); - if (_head.getFaceModel().isActive()) { - skinColor = glm::vec3(_head.getFaceModel().computeAverageColor()); - const float SKIN_DARKENING = 0.9f; - darkSkinColor = skinColor * SKIN_DARKENING; - } + glm::vec3 skinColor, darkSkinColor; + getSkinColors(skinColor, darkSkinColor); for (int b = 0; b < NUM_AVATAR_BODY_BALLS; b++) { float alpha = getBallRenderAlpha(b, lookingInMirror); @@ -829,6 +824,16 @@ void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { _hand.render(lookingInMirror); } +void Avatar::getSkinColors(glm::vec3& lighter, glm::vec3& darker) { + lighter = glm::vec3(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2]); + darker = glm::vec3(DARK_SKIN_COLOR[0], DARK_SKIN_COLOR[1], DARK_SKIN_COLOR[2]); + if (_head.getFaceModel().isActive()) { + lighter = glm::vec3(_head.getFaceModel().computeAverageColor()); + const float SKIN_DARKENING = 0.9f; + darker = lighter * SKIN_DARKENING; + } +} + void Avatar::getBodyBallTransform(AvatarJointID jointID, glm::vec3& position, glm::quat& rotation) const { position = _bodyBall[jointID].position; rotation = _bodyBall[jointID].rotation; diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index 50e9eefdc0..f1b25ea640 100755 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -158,6 +158,8 @@ public: glm::quat getWorldAlignedOrientation() const; AvatarVoxelSystem* getVoxels() { return &_voxels; } + void getSkinColors(glm::vec3& lighter, glm::vec3& darker); + // Get the position/rotation of a single body ball void getBodyBallTransform(AvatarJointID jointID, glm::vec3& position, glm::quat& rotation) const; diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index b32af7dba6..cccbb4c8b6 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -617,13 +617,8 @@ void MyAvatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { } } else if (renderAvatarBalls || !(_voxels.getVoxelURL().isValid() || _skeletonModel.isActive())) { // Render the body as balls and cones - glm::vec3 skinColor(SKIN_COLOR[0], SKIN_COLOR[1], SKIN_COLOR[2]); - glm::vec3 darkSkinColor(DARK_SKIN_COLOR[0], DARK_SKIN_COLOR[1], DARK_SKIN_COLOR[2]); - if (_head.getFaceModel().isActive()) { - skinColor = glm::vec3(_head.getFaceModel().computeAverageColor()); - const float SKIN_DARKENING = 0.9f; - darkSkinColor = skinColor * SKIN_DARKENING; - } + glm::vec3 skinColor, darkSkinColor; + getSkinColors(skinColor, darkSkinColor); for (int b = 0; b < NUM_AVATAR_BODY_BALLS; b++) { float alpha = getBallRenderAlpha(b, lookingInMirror); diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 7ea1572089..8c2bd675e5 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -36,20 +36,38 @@ bool SkeletonModel::render(float alpha) { const FBXGeometry& geometry = _geometry->getFBXGeometry(); + glm::vec3 skinColor, darkSkinColor; + _owningAvatar->getSkinColors(skinColor, darkSkinColor); + for (int i = 0; i < _jointStates.size(); i++) { - const JointState& joint = _jointStates.at(i); - glPushMatrix(); - glMultMatrixf((const GLfloat*)&joint.transform); - if (i == geometry.rootJointIndex) { - glColor3f(1.0f, 0.0f, 0.0f); - } else { - glColor3f(1.0f, 1.0f, 1.0f); + glm::vec3 position; + getJointPosition(i, position); + glTranslatef(position.x, position.y, position.z); + + glm::quat rotation; + getJointRotation(i, rotation); + glm::vec3 axis = glm::axis(rotation); + glRotatef(glm::angle(rotation), axis.x, axis.y, axis.z); + + glColor4f(skinColor.r, skinColor.g, skinColor.b, alpha); + const float BALL_RADIUS = 0.02f; + const int BALL_SUBDIVISIONS = 10; + glutSolidSphere(BALL_RADIUS * _owningAvatar->getScale(), BALL_SUBDIVISIONS, BALL_SUBDIVISIONS); + + glPopMatrix(); + + int parentIndex = geometry.joints[i].parentIndex; + if (parentIndex == -1) { + continue; } - glutSolidSphere(0.05f, 10, 10); + glColor4f(darkSkinColor.r, darkSkinColor.g, darkSkinColor.b, alpha); - glPopMatrix(); + glm::vec3 parentPosition; + getJointPosition(parentIndex, parentPosition); + const float STICK_RADIUS = BALL_RADIUS * 0.5f; + Avatar::renderJointConnectingCone(parentPosition, position, STICK_RADIUS, STICK_RADIUS); } Model::render(alpha); diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index 3bd1b2fa55..19864c3b31 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -98,11 +98,11 @@ protected: virtual void maybeUpdateNeckRotation(const JointState& parentState, const FBXJoint& joint, JointState& state); virtual void maybeUpdateEyeRotation(const JointState& parentState, const FBXJoint& joint, JointState& state); -private: - bool getJointPosition(int jointIndex, glm::vec3& position) const; bool getJointRotation(int jointIndex, glm::quat& rotation) const; +private: + void deleteGeometry(); float _pupilDilation; From cf98d900f98d318f0915a4478c6872d7b8265dc4 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 22 Oct 2013 17:26:42 -0700 Subject: [PATCH 25/27] Removed unused variable. --- interface/src/renderer/Model.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 525f73e196..39f1df1654 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -66,7 +66,6 @@ void Model::simulate(float deltaTime) { // set up world vertices on first simulate after load const FBXGeometry& geometry = _geometry->getFBXGeometry(); if (_jointStates.isEmpty()) { - QVector vertices; foreach (const FBXJoint& joint, geometry.joints) { JointState state; state.rotation = joint.rotation; From e8885332f2583af6fdb6e230ed0feb9d3fcd95f6 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 23 Oct 2013 11:39:30 -0700 Subject: [PATCH 26/27] Code review fixes. --- interface/src/DataServerClient.cpp | 14 +++++++------- interface/src/avatar/SkeletonModel.cpp | 3 +-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/interface/src/DataServerClient.cpp b/interface/src/DataServerClient.cpp index 6fe864d13a..b3d8592027 100644 --- a/interface/src/DataServerClient.cpp +++ b/interface/src/DataServerClient.cpp @@ -131,8 +131,8 @@ void DataServerClient::processSendFromDataServer(unsigned char* packetData, int if (keyList[i] == DataServerKey::FaceMeshURL) { if (userUUID.isNull() || userUUID == Application::getInstance()->getProfile()->getUUID()) { - qDebug("Changing user's face model URL to %s\n", valueList[0].toLocal8Bit().constData()); - Application::getInstance()->getProfile()->setFaceModelURL(QUrl(valueList[0])); + qDebug("Changing user's face model URL to %s\n", valueList[i].toLocal8Bit().constData()); + Application::getInstance()->getProfile()->setFaceModelURL(QUrl(valueList[i])); } else { // mesh URL for a UUID, find avatar in our list NodeList* nodeList = NodeList::getInstance(); @@ -142,7 +142,7 @@ void DataServerClient::processSendFromDataServer(unsigned char* packetData, int if (avatar->getUUID() == userUUID) { QMetaObject::invokeMethod(&avatar->getHead().getFaceModel(), - "setURL", Q_ARG(QUrl, QUrl(valueList[0]))); + "setURL", Q_ARG(QUrl, QUrl(valueList[i]))); } } } @@ -150,8 +150,8 @@ void DataServerClient::processSendFromDataServer(unsigned char* packetData, int } else if (keyList[i] == DataServerKey::SkeletonURL) { if (userUUID.isNull() || userUUID == Application::getInstance()->getProfile()->getUUID()) { - qDebug("Changing user's skeleton URL to %s\n", valueList[0].toLocal8Bit().constData()); - Application::getInstance()->getProfile()->setSkeletonModelURL(QUrl(valueList[0])); + qDebug("Changing user's skeleton URL to %s\n", valueList[i].toLocal8Bit().constData()); + Application::getInstance()->getProfile()->setSkeletonModelURL(QUrl(valueList[i])); } else { // skeleton URL for a UUID, find avatar in our list NodeList* nodeList = NodeList::getInstance(); @@ -161,7 +161,7 @@ void DataServerClient::processSendFromDataServer(unsigned char* packetData, int if (avatar->getUUID() == userUUID) { QMetaObject::invokeMethod(&avatar->getSkeletonModel(), "setURL", - Q_ARG(QUrl, QUrl(valueList[0]))); + Q_ARG(QUrl, QUrl(valueList[i]))); } } } @@ -187,7 +187,7 @@ void DataServerClient::processSendFromDataServer(unsigned char* packetData, int } else if (keyList[i] == DataServerKey::UUID) { // this is the user's UUID - set it on the profile - Application::getInstance()->getProfile()->setUUID(valueList[0]); + Application::getInstance()->getProfile()->setUUID(valueList[i]); } } } diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 8c2bd675e5..9ccc63d587 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -12,8 +12,7 @@ #include "SkeletonModel.h" SkeletonModel::SkeletonModel(Avatar* owningAvatar) : - _owningAvatar(owningAvatar) -{ + _owningAvatar(owningAvatar) { } void SkeletonModel::simulate(float deltaTime) { From 183a866e9eb09d0a99f5abbc33467ea30f1f9448 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 23 Oct 2013 11:44:28 -0700 Subject: [PATCH 27/27] Remove code that will be added back in fullscreen mirror request. --- interface/src/Application.cpp | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 330035ad2a..fef149aac1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -367,20 +367,7 @@ void Application::paintGL() { glEnable(GL_LINE_SMOOTH); - if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { - _myCamera.setTightness (100.0f); - glm::vec3 targetPosition = _myAvatar.getUprightHeadPosition(); - if (_myAvatar.getHead().getFaceModel().isActive()) { - // make sure we're aligned to the blend face eyes - glm::vec3 leftEyePosition, rightEyePosition; - if (_myAvatar.getHead().getFaceModel().getEyePositions(leftEyePosition, rightEyePosition)) { - targetPosition = (leftEyePosition + rightEyePosition) * 0.5f; - } - } - _myCamera.setTargetPosition(targetPosition); - _myCamera.setTargetRotation(_myAvatar.getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PIf, 0.0f))); - - } else if (OculusManager::isConnected()) { + if (OculusManager::isConnected()) { _myCamera.setUpShift (0.0f); _myCamera.setDistance (0.0f); _myCamera.setTightness (0.0f); // Camera is directly connected to head without smoothing