From 635f3b6dc387fd6bd1e47cc14bb750f2cfd4eaa3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Mar 2014 16:38:05 -0700 Subject: [PATCH] Perform the mesh blending in worker threads and only when we've actually received new data. Closes #2075. Closes #2348. --- interface/src/avatar/MyAvatar.cpp | 5 - interface/src/devices/Faceshift.cpp | 4 +- interface/src/devices/Faceshift.h | 8 +- interface/src/devices/Visage.h | 6 +- interface/src/renderer/GeometryCache.cpp | 8 ++ interface/src/renderer/GeometryCache.h | 12 +- interface/src/renderer/Model.cpp | 161 ++++++++++++++++------- interface/src/renderer/Model.h | 20 +-- interface/src/renderer/TextureCache.h | 2 +- libraries/avatars/src/HeadData.h | 7 +- 10 files changed, 155 insertions(+), 78 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 6c51026097..fb0d704c6a 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -424,11 +424,6 @@ void MyAvatar::updateFromGyros(float deltaTime) { } } -static TextRenderer* textRenderer() { - static TextRenderer* renderer = new TextRenderer(SANS_FONT_FAMILY, 24, -1, false, TextRenderer::SHADOW_EFFECT); - return renderer; -} - void MyAvatar::renderDebugBodyPoints() { glm::vec3 torsoPosition(getPosition()); glm::vec3 headPosition(getHead()->getEyePosition()); diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 88974ce493..9f1734c7e5 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -103,7 +103,7 @@ void Faceshift::reset() { } void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen, std::vector& coefficients) const { + float jawOpen, QVector& coefficients) const { coefficients.resize(max((int)coefficients.size(), _jawOpenIndex + 1)); qFill(coefficients.begin(), coefficients.end(), 0.0f); coefficients[_leftBlinkIndex] = leftBlink; @@ -204,7 +204,7 @@ void Faceshift::receive(const QByteArray& buffer) { _eyeGazeLeftYaw = data.m_eyeGazeLeftYaw; _eyeGazeRightPitch = -data.m_eyeGazeRightPitch; _eyeGazeRightYaw = data.m_eyeGazeRightYaw; - _blendshapeCoefficients = data.m_coeffs; + _blendshapeCoefficients = QVector::fromStdVector(data.m_coeffs); _lastTrackingStateReceived = usecTimestampNow(); } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index a0898c446d..f878056b57 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -9,8 +9,6 @@ #ifndef __interface__Faceshift__ #define __interface__Faceshift__ -#include - #include #include @@ -47,7 +45,7 @@ public: float getEstimatedEyePitch() const { return _estimatedEyePitch; } float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } - const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); } float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); } @@ -68,7 +66,7 @@ public: void reset(); void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp, - float jawOpen, std::vector& coefficients) const; + float jawOpen, QVector& coefficients) const; signals: @@ -111,7 +109,7 @@ private: float _eyeGazeRightPitch; float _eyeGazeRightYaw; - std::vector _blendshapeCoefficients; + QVector _blendshapeCoefficients; int _leftBlinkIndex; int _rightBlinkIndex; diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h index 7e50812ba7..6e98abbb61 100644 --- a/interface/src/devices/Visage.h +++ b/interface/src/devices/Visage.h @@ -9,8 +9,6 @@ #ifndef __interface__Visage__ #define __interface__Visage__ -#include - #include #include #include @@ -42,7 +40,7 @@ public: float getEstimatedEyePitch() const { return _estimatedEyePitch; } float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } - const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } void update(); void reset(); @@ -71,7 +69,7 @@ private: float _estimatedEyePitch; float _estimatedEyeYaw; - std::vector _blendshapeCoefficients; + QVector _blendshapeCoefficients; }; #endif /* defined(__interface__Visage__) */ diff --git a/interface/src/renderer/GeometryCache.cpp b/interface/src/renderer/GeometryCache.cpp index bd5915397b..c4a0d15baa 100644 --- a/interface/src/renderer/GeometryCache.cpp +++ b/interface/src/renderer/GeometryCache.cpp @@ -13,6 +13,7 @@ #include "Application.h" #include "GeometryCache.h" +#include "Model.h" #include "world.h" GeometryCache::~GeometryCache() { @@ -291,6 +292,13 @@ QSharedPointer GeometryCache::getGeometry(const QUrl& url, cons return getResource(url, fallback, delayLoad).staticCast(); } +void GeometryCache::setBlendedVertices(const QPointer& model, const QWeakPointer& geometry, + const QVector& vertices, const QVector& normals) { + if (!model.isNull() && model->getGeometry() == geometry) { + model->setBlendedVertices(vertices, normals); + } +} + QSharedPointer GeometryCache::createResource(const QUrl& url, const QSharedPointer& fallback, bool delayLoad, const void* extra) { diff --git a/interface/src/renderer/GeometryCache.h b/interface/src/renderer/GeometryCache.h index cc0775051a..252a0c401b 100644 --- a/interface/src/renderer/GeometryCache.h +++ b/interface/src/renderer/GeometryCache.h @@ -19,15 +19,18 @@ #include "FBXReader.h" +class Model; class NetworkGeometry; class NetworkMesh; class NetworkTexture; /// Stores cached geometry. class GeometryCache : public ResourceCache { + Q_OBJECT + public: - ~GeometryCache(); + virtual ~GeometryCache(); void renderHemisphere(int slices, int stacks); void renderSquare(int xDivisions, int yDivisions); @@ -38,7 +41,12 @@ public: /// \param fallback a fallback URL to load if the desired one is unavailable /// \param delayLoad if true, don't load the geometry immediately; wait until load is first requested QSharedPointer getGeometry(const QUrl& url, const QUrl& fallback = QUrl(), bool delayLoad = false); - + +public slots: + + void setBlendedVertices(const QPointer& model, const QWeakPointer& geometry, + const QVector& vertices, const QVector& normals); + protected: virtual QSharedPointer createResource(const QUrl& url, diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index df6d41ebbc..591cc4a021 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -6,6 +6,10 @@ // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // +#include +#include +#include + #include #include @@ -19,6 +23,10 @@ using namespace std; +static int modelPointerTypeId = qRegisterMetaType >(); +static int weakNetworkGeometryPointerTypeId = qRegisterMetaType >(); +static int vec3VectorTypeId = qRegisterMetaType >(); + Model::Model(QObject* parent) : QObject(parent), _scale(1.0f, 1.0f, 1.0f), @@ -104,8 +112,6 @@ void Model::init() { } void Model::reset() { - _resetStates = true; - foreach (Model* attachment, _attachments) { attachment->reset(); } @@ -170,20 +176,10 @@ bool Model::render(float alpha) { return false; } - // set up blended buffer ids on first render after load/simulate + // set up dilated textures on first render after load/simulate const FBXGeometry& geometry = _geometry->getFBXGeometry(); - if (_blendedVertexBufferIDs.isEmpty()) { + if (_dilatedTextures.isEmpty()) { foreach (const FBXMesh& mesh, geometry.meshes) { - GLuint id = 0; - if (!mesh.blendshapes.isEmpty()) { - glGenBuffers(1, &id); - glBindBuffer(GL_ARRAY_BUFFER, id); - glBufferData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3), - NULL, GL_DYNAMIC_DRAW); - glBindBuffer(GL_ARRAY_BUFFER, 0); - } - _blendedVertexBufferIDs.append(id); - QVector > dilated; dilated.resize(mesh.parts.size()); _dilatedTextures.append(dilated); @@ -478,6 +474,68 @@ QVector Model::updateGeometry() { return newJointStates; } +class Blender : public QRunnable { +public: + + Blender(Model* model, const QWeakPointer& geometry, + const QVector& meshes, const QVector& blendshapeCoefficients); + + virtual void run(); + +private: + + QPointer _model; + QWeakPointer _geometry; + QVector _meshes; + QVector _blendshapeCoefficients; +}; + +Blender::Blender(Model* model, const QWeakPointer& geometry, + const QVector& meshes, const QVector& blendshapeCoefficients) : + _model(model), + _geometry(geometry), + _meshes(meshes), + _blendshapeCoefficients(blendshapeCoefficients) { +} + +void Blender::run() { + // make sure the model/geometry still exists + if (_model.isNull() || _geometry.isNull()) { + return; + } + QVector vertices, normals; + int offset = 0; + foreach (const FBXMesh& mesh, _meshes) { + if (mesh.blendshapes.isEmpty()) { + continue; + } + vertices += mesh.vertices; + normals += mesh.normals; + glm::vec3* meshVertices = vertices.data() + offset; + glm::vec3* meshNormals = normals.data() + offset; + offset += mesh.vertices.size(); + const float NORMAL_COEFFICIENT_SCALE = 0.01f; + for (int i = 0, n = qMin(_blendshapeCoefficients.size(), mesh.blendshapes.size()); i < n; i++) { + float vertexCoefficient = _blendshapeCoefficients.at(i); + if (vertexCoefficient < EPSILON) { + continue; + } + float normalCoefficient = vertexCoefficient * NORMAL_COEFFICIENT_SCALE; + const FBXBlendshape& blendshape = mesh.blendshapes.at(i); + for (int j = 0; j < blendshape.indices.size(); j++) { + int index = blendshape.indices.at(j); + meshVertices[index] += blendshape.vertices.at(j) * vertexCoefficient; + meshNormals[index] += blendshape.normals.at(j) * normalCoefficient; + } + } + } + + // post the result to the geometry cache, which will dispatch to the model if still alive + QMetaObject::invokeMethod(Application::getInstance()->getGeometryCache(), "setBlendedVertices", + Q_ARG(const QPointer&, _model), Q_ARG(const QWeakPointer&, _geometry), + Q_ARG(const QVector&, vertices), Q_ARG(const QVector&, normals)); +} + void Model::simulate(float deltaTime, bool fullUpdate, const QVector& newJointStates) { if (!isActive()) { return; @@ -491,6 +549,19 @@ void Model::simulate(float deltaTime, bool fullUpdate, const QVector MeshState state; state.clusterMatrices.resize(mesh.clusters.size()); _meshStates.append(state); + + QOpenGLBuffer buffer; + if (!mesh.blendshapes.isEmpty()) { + buffer.setUsagePattern(QOpenGLBuffer::DynamicDraw); + buffer.create(); + buffer.bind(); + buffer.allocate((mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3)); + buffer.write(0, mesh.vertices.constData(), mesh.vertices.size() * sizeof(glm::vec3)); + buffer.write(mesh.vertices.size() * sizeof(glm::vec3), mesh.normals.constData(), + mesh.normals.size() * sizeof(glm::vec3)); + buffer.release(); + } + _blendedVertexBuffers.append(buffer); } foreach (const FBXAttachment& attachment, geometry.attachments) { Model* model = new Model(this); @@ -498,12 +569,12 @@ void Model::simulate(float deltaTime, bool fullUpdate, const QVector model->setURL(attachment.url); _attachments.append(model); } - _resetStates = fullUpdate = true; + fullUpdate = true; createCollisionShapes(); } // exit early if we don't have to perform a full update - if (!(fullUpdate || _resetStates)) { + if (!fullUpdate) { return; } @@ -537,7 +608,9 @@ void Model::simulate(float deltaTime, bool fullUpdate, const QVector state.clusterMatrices[j] = _jointStates[cluster.jointIndex].transform * cluster.inverseBindMatrix; } } - _resetStates = false; + + // post the blender + QThreadPool::globalInstance()->start(new Blender(this, _geometry, geometry.meshes, _blendshapeCoefficients)); } void Model::updateJointState(int index) { @@ -836,6 +909,27 @@ void Model::applyCollision(CollisionInfo& collision) { } } +void Model::setBlendedVertices(const QVector& vertices, const QVector& normals) { + if (_blendedVertexBuffers.isEmpty()) { + return; + } + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + int index = 0; + for (int i = 0; i < geometry.meshes.size(); i++) { + const FBXMesh& mesh = geometry.meshes.at(i); + if (mesh.blendshapes.isEmpty()) { + continue; + } + QOpenGLBuffer& buffer = _blendedVertexBuffers[i]; + buffer.bind(); + buffer.write(0, vertices.constData() + index, mesh.vertices.size() * sizeof(glm::vec3)); + buffer.write(mesh.vertices.size() * sizeof(glm::vec3), normals.constData() + index, + mesh.normals.size() * sizeof(glm::vec3)); + buffer.release(); + index += mesh.vertices.size(); + } +} + void Model::applyNextGeometry() { // delete our local geometry and custom textures deleteGeometry(); @@ -854,10 +948,7 @@ void Model::deleteGeometry() { delete attachment; } _attachments.clear(); - foreach (GLuint id, _blendedVertexBufferIDs) { - glDeleteBuffers(1, &id); - } - _blendedVertexBufferIDs.clear(); + _blendedVertexBuffers.clear(); _jointStates.clear(); _meshStates.clear(); clearShapes(); @@ -941,33 +1032,7 @@ void Model::renderMeshes(float alpha, bool translucent) { } glColorPointer(3, GL_FLOAT, 0, (void*)(mesh.tangents.size() * sizeof(glm::vec3))); glTexCoordPointer(2, GL_FLOAT, 0, (void*)((mesh.tangents.size() + mesh.colors.size()) * sizeof(glm::vec3))); - glBindBuffer(GL_ARRAY_BUFFER, _blendedVertexBufferIDs.at(i)); - - _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); - _blendedNormals.resize(_blendedVertices.size()); - memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3)); - memcpy(_blendedNormals.data(), mesh.normals.constData(), vertexCount * sizeof(glm::vec3)); - - // blend in each coefficient - for (unsigned int j = 0; j < _blendshapeCoefficients.size(); j++) { - float coefficient = _blendshapeCoefficients[j]; - if (coefficient == 0.0f || j >= (unsigned int)mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) { - continue; - } - const float NORMAL_COEFFICIENT_SCALE = 0.01f; - float normalCoefficient = coefficient * NORMAL_COEFFICIENT_SCALE; - const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData(); - const glm::vec3* normal = mesh.blendshapes[j].normals.constData(); - for (const int* index = mesh.blendshapes[j].indices.constData(), - *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++, normal++) { - _blendedVertices[*index] += *vertex * coefficient; - _blendedNormals[*index] += *normal * normalCoefficient; - } - } - - glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), _blendedVertices.constData()); - glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3), - vertexCount * sizeof(glm::vec3), _blendedNormals.constData()); + _blendedVertexBuffers[i].bind(); } glVertexPointer(3, GL_FLOAT, 0, 0); glNormalPointer(GL_FLOAT, 0, (void*)(vertexCount * sizeof(glm::vec3))); diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index 5b1fbb3eea..f08a6b9fc2 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -43,8 +43,8 @@ public: void setPupilDilation(float dilation) { _pupilDilation = dilation; } float getPupilDilation() const { return _pupilDilation; } - void setBlendshapeCoefficients(const std::vector& coefficients) { _blendshapeCoefficients = coefficients; } - const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + void setBlendshapeCoefficients(const QVector& coefficients) { _blendshapeCoefficients = coefficients; } + const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } bool isActive() const { return _geometry && _geometry->isLoaded(); } @@ -195,6 +195,9 @@ public: /// Use the collision to affect the model void applyCollision(CollisionInfo& collision); + /// Sets blended vertices computed in a separate thread. + void setBlendedVertices(const QVector& vertices, const QVector& normals); + protected: QSharedPointer _geometry; @@ -268,16 +271,13 @@ private: float _nextLODHysteresis; float _pupilDilation; - std::vector _blendshapeCoefficients; + QVector _blendshapeCoefficients; QUrl _url; - QVector _blendedVertexBufferIDs; - QVector > > _dilatedTextures; - bool _resetStates; + QVector _blendedVertexBuffers; - QVector _blendedVertices; - QVector _blendedNormals; + QVector > > _dilatedTextures; QVector _attachments; @@ -303,4 +303,8 @@ private: static QVector createJointStates(const FBXGeometry& geometry); }; +Q_DECLARE_METATYPE(QPointer) +Q_DECLARE_METATYPE(QWeakPointer) +Q_DECLARE_METATYPE(QVector) + #endif /* defined(__interface__Model__) */ diff --git a/interface/src/renderer/TextureCache.h b/interface/src/renderer/TextureCache.h index d7e61954ce..ebeb35119c 100644 --- a/interface/src/renderer/TextureCache.h +++ b/interface/src/renderer/TextureCache.h @@ -27,7 +27,7 @@ class TextureCache : public ResourceCache { public: TextureCache(); - ~TextureCache(); + virtual ~TextureCache(); /// Returns the ID of the permutation/normal texture used for Perlin noise shader programs. This texture /// has two lines: the first, a set of random numbers in [0, 255] to be used as permutation offsets, and diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 56dc5630d0..b199ff19d2 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -10,7 +10,8 @@ #define __hifi__HeadData__ #include -#include + +#include #include #include @@ -54,7 +55,7 @@ public: float getAudioAverageLoudness() const { return _audioAverageLoudness; } void setAudioAverageLoudness(float audioAverageLoudness) { _audioAverageLoudness = audioAverageLoudness; } - const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } float getPupilDilation() const { return _pupilDilation; } void setPupilDilation(float pupilDilation) { _pupilDilation = pupilDilation; } @@ -86,7 +87,7 @@ protected: float _averageLoudness; float _browAudioLift; float _audioAverageLoudness; - std::vector _blendshapeCoefficients; + QVector _blendshapeCoefficients; float _pupilDilation; AvatarData* _owningAvatar;