From 264f41472d1eb3db04858f03b6f81bd484175596 Mon Sep 17 00:00:00 2001 From: Olivier Prat Date: Fri, 15 Dec 2017 12:06:07 +0100 Subject: [PATCH] Added tangents to blendshape for possible break of bump mapping when doing blend shape animations --- libraries/fbx/src/FBX.h | 1 + libraries/fbx/src/FBXReader.cpp | 96 +++++++++++++++---- libraries/fbx/src/FBXReader_Mesh.cpp | 8 ++ .../render-utils/src/MeshPartPayload.cpp | 3 +- libraries/render-utils/src/Model.cpp | 96 ++++++++++++------- libraries/render-utils/src/Model.h | 4 +- 6 files changed, 151 insertions(+), 57 deletions(-) diff --git a/libraries/fbx/src/FBX.h b/libraries/fbx/src/FBX.h index 7d3328a2dd..2150463065 100644 --- a/libraries/fbx/src/FBX.h +++ b/libraries/fbx/src/FBX.h @@ -60,6 +60,7 @@ public: QVector indices; QVector vertices; QVector normals; + QVector tangents; }; struct FBXJointShapeInfo { diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index a5d15350d5..1694e31e1d 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -303,36 +303,45 @@ FBXBlendshape extractBlendshape(const FBXNode& object) { return blendshape; } -static void setTangents(FBXMesh& mesh, int firstIndex, int secondIndex) { - const glm::vec3& normal = mesh.normals.at(firstIndex); - glm::vec3 bitangent = glm::cross(normal, mesh.vertices.at(secondIndex) - mesh.vertices.at(firstIndex)); - if (glm::length(bitangent) < EPSILON) { - return; +using IndexAccessor = std::function; + +static void setTangents(const FBXMesh& mesh, const IndexAccessor& vertexAccessor, int firstIndex, int secondIndex, + const QVector& vertices, const QVector& normals, QVector& tangents) { + glm::vec3 vertex[2]; + glm::vec3 normal; + glm::vec3* tangent = vertexAccessor(mesh, firstIndex, secondIndex, vertex, normal); + if (tangent) { + glm::vec3 bitangent = glm::cross(normal, vertex[1] - vertex[0]); + if (glm::length(bitangent) < EPSILON) { + return; + } + glm::vec2 texCoordDelta = mesh.texCoords.at(secondIndex) - mesh.texCoords.at(firstIndex); + glm::vec3 normalizedNormal = glm::normalize(normal); + *tangent += glm::cross(glm::angleAxis(-atan2f(-texCoordDelta.t, texCoordDelta.s), normalizedNormal) * + glm::normalize(bitangent), normalizedNormal); } - glm::vec2 texCoordDelta = mesh.texCoords.at(secondIndex) - mesh.texCoords.at(firstIndex); - glm::vec3 normalizedNormal = glm::normalize(normal); - mesh.tangents[firstIndex] += glm::cross(glm::angleAxis(-atan2f(-texCoordDelta.t, texCoordDelta.s), normalizedNormal) * - glm::normalize(bitangent), normalizedNormal); } -static void createTangents(FBXMesh& mesh, bool generateFromTexCoords) { +static void createTangents(const FBXMesh& mesh, bool generateFromTexCoords, + const QVector& vertices, const QVector& normals, QVector& tangents, + IndexAccessor accessor) { // if we have a normal map (and texture coordinates), we must compute tangents if (generateFromTexCoords && !mesh.texCoords.isEmpty()) { - mesh.tangents.resize(mesh.vertices.size()); + tangents.resize(vertices.size()); foreach(const FBXMeshPart& part, mesh.parts) { for (int i = 0; i < part.quadIndices.size(); i += 4) { - setTangents(mesh, part.quadIndices.at(i), part.quadIndices.at(i + 1)); - setTangents(mesh, part.quadIndices.at(i + 1), part.quadIndices.at(i + 2)); - setTangents(mesh, part.quadIndices.at(i + 2), part.quadIndices.at(i + 3)); - setTangents(mesh, part.quadIndices.at(i + 3), part.quadIndices.at(i)); + setTangents(mesh, accessor, part.quadIndices.at(i), part.quadIndices.at(i + 1), vertices, normals, tangents); + setTangents(mesh, accessor, part.quadIndices.at(i + 1), part.quadIndices.at(i + 2), vertices, normals, tangents); + setTangents(mesh, accessor, part.quadIndices.at(i + 2), part.quadIndices.at(i + 3), vertices, normals, tangents); + setTangents(mesh, accessor, part.quadIndices.at(i + 3), part.quadIndices.at(i), vertices, normals, tangents); } // <= size - 3 in order to prevent overflowing triangleIndices when (i % 3) != 0 // This is most likely evidence of a further problem in extractMesh() for (int i = 0; i <= part.triangleIndices.size() - 3; i += 3) { - setTangents(mesh, part.triangleIndices.at(i), part.triangleIndices.at(i + 1)); - setTangents(mesh, part.triangleIndices.at(i + 1), part.triangleIndices.at(i + 2)); - setTangents(mesh, part.triangleIndices.at(i + 2), part.triangleIndices.at(i)); + setTangents(mesh, accessor, part.triangleIndices.at(i), part.triangleIndices.at(i + 1), vertices, normals, tangents); + setTangents(mesh, accessor, part.triangleIndices.at(i + 1), part.triangleIndices.at(i + 2), vertices, normals, tangents); + setTangents(mesh, accessor, part.triangleIndices.at(i + 2), part.triangleIndices.at(i), vertices, normals, tangents); } if ((part.triangleIndices.size() % 3) != 0) { qCDebug(modelformat) << "Error in extractFBXGeometry part.triangleIndices.size() is not divisible by three "; @@ -341,6 +350,52 @@ static void createTangents(FBXMesh& mesh, bool generateFromTexCoords) { } } +static void createMeshTangents(FBXMesh& mesh, bool generateFromTexCoords) { + // This is the only workaround I've found to trick the compiler into understanding that mesh.tangents isn't + // const in the lambda function. + auto& tangents = mesh.tangents; + createTangents(mesh, generateFromTexCoords, mesh.vertices, mesh.normals, mesh.tangents, + [&](const FBXMesh& mesh, int firstIndex, int secondIndex, glm::vec3* outVertices, glm::vec3& outNormal) { + outVertices[0] = mesh.vertices[firstIndex]; + outVertices[1] = mesh.vertices[secondIndex]; + outNormal = mesh.normals[firstIndex]; + return &(tangents[firstIndex]); + }); +} + +static void createBlendShapeTangents(FBXMesh& mesh, bool generateFromTexCoords, FBXBlendshape& blendShape) { + // Create lookup to get index in blend shape from vertex index in mesh + std::vector reverseIndices; + reverseIndices.resize(mesh.vertices.size()); + std::iota(reverseIndices.begin(), reverseIndices.end(), 0); + + for (int indexInBlendShape = 0; indexInBlendShape < blendShape.indices.size(); ++indexInBlendShape) { + auto indexInMesh = blendShape.indices[indexInBlendShape]; + reverseIndices[indexInMesh] = indexInBlendShape; + } + + createTangents(mesh, generateFromTexCoords, blendShape.vertices, blendShape.normals, blendShape.tangents, + [&](const FBXMesh& mesh, int firstIndex, int secondIndex, glm::vec3* outVertices, glm::vec3& outNormal) { + const auto index1 = reverseIndices[firstIndex]; + const auto index2 = reverseIndices[secondIndex]; + + if (index1 < blendShape.vertices.size()) { + outVertices[0] = blendShape.vertices[index1]; + if (index2 < blendShape.vertices.size()) { + outVertices[1] = blendShape.vertices[index2]; + } else { + // Index isn't in the blend shape so return vertex from mesh + outVertices[1] = mesh.vertices[secondIndex]; + } + outNormal = blendShape.normals[index1]; + return &blendShape.tangents[index1]; + } else { + // Index isn't in blend shape so return nullptr + return (glm::vec3*)nullptr; + } + }); +} + QVector getIndices(const QVector ids, QVector modelIDs) { QVector indices; foreach (const QString& id, ids) { @@ -1595,7 +1650,10 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS } } - createTangents(extracted.mesh, generateTangents); + createMeshTangents(extracted.mesh, generateTangents); + for (auto& blendShape : extracted.mesh.blendshapes) { + createBlendShapeTangents(extracted.mesh, generateTangents, blendShape); + } // find the clusters with which the mesh is associated QVector clusterIDs; diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 2f307f0afa..fbe74ccf99 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -589,6 +589,14 @@ void FBXReader::buildModelMesh(FBXMesh& extractedMesh, const QString& url) { fbxMesh.tangents.reserve(fbxMesh.normals.size()); std::fill_n(std::back_inserter(fbxMesh.tangents), fbxMesh.normals.size(), Vectors::UNIT_X); } + // Same thing with blend shapes + for (auto& blendShape : fbxMesh.blendshapes) { + if (!blendShape.normals.empty() && blendShape.tangents.empty()) { + // Fill with a dummy value to force tangents to be present if there are normals + blendShape.tangents.reserve(blendShape.normals.size()); + std::fill_n(std::back_inserter(fbxMesh.tangents), blendShape.normals.size(), Vectors::UNIT_X); + } + } // evaluate all attribute channels sizes const int normalsSize = fbxMesh.normals.size() * sizeof(NormalType); diff --git a/libraries/render-utils/src/MeshPartPayload.cpp b/libraries/render-utils/src/MeshPartPayload.cpp index 96451fa23d..c3534da114 100644 --- a/libraries/render-utils/src/MeshPartPayload.cpp +++ b/libraries/render-utils/src/MeshPartPayload.cpp @@ -518,7 +518,8 @@ void ModelMeshPartPayload::bindMesh(gpu::Batch& batch) { ModelPointer model = _model.lock(); if (model) { batch.setInputBuffer(0, model->_blendedVertexBuffers[_meshIndex], 0, sizeof(glm::vec3)); - batch.setInputBuffer(1, model->_blendedVertexBuffers[_meshIndex], _drawMesh->getNumVertices() * sizeof(glm::vec3), sizeof(NormalType)); + // Stride is 2*sizeof(glm::vec3) because normal and tangents are interleaved + batch.setInputBuffer(1, model->_blendedVertexBuffers[_meshIndex], _drawMesh->getNumVertices() * sizeof(glm::vec3), 2*sizeof(NormalType)); batch.setInputStream(2, _drawMesh->getVertexStream().makeRangedStream(2)); } else { batch.setIndexBuffer(gpu::UINT32, (_drawMesh->getIndexBuffer()._buffer), 0); diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index d918970af5..e341fb886e 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -317,21 +317,29 @@ bool Model::updateGeometry() { // TODO? make _blendedVertexBuffers a map instead of vector and only add for meshes with blendshapes? auto buffer = std::make_shared(); if (!mesh.blendshapes.isEmpty()) { - buffer->resize(mesh.vertices.size() * sizeof(glm::vec3) + mesh.normals.size() * sizeof(NormalType)); - buffer->setSubData(0, mesh.vertices.size() * sizeof(glm::vec3), (const gpu::Byte*) mesh.vertices.constData()); + std::vector normalsAndTangents; + normalsAndTangents.reserve(mesh.normals.size() + mesh.tangents.size()); + + for (auto normalIt = mesh.normals.begin(), tangentIt = mesh.tangents.begin(); + normalIt != mesh.normals.end(); + ++normalIt, ++tangentIt) { #if FBX_PACK_NORMALS - std::vector packedNormals; - packedNormals.reserve(mesh.normals.size()); - for (auto normal : mesh.normals) { - normal = FBXReader::normalizeDirForPacking(normal); - packedNormals.push_back(glm::packSnorm3x10_1x2(glm::vec4(normal, 0.0f))); - } - const auto normalsData = packedNormals.data(); + const auto normal = FBXReader::normalizeDirForPacking(*normalIt); + const auto tangent = FBXReader::normalizeDirForPacking(*tangentIt); + const auto finalNormal = glm::packSnorm3x10_1x2(glm::vec4(normal, 0.0f)); + const auto finalTangent = glm::packSnorm3x10_1x2(glm::vec4(tangent, 0.0f)); #else - const auto normalsData = mesh.normals.constData(); + const auto finalNormal = *normalIt; + const auto finalTangent = *tangentIt; #endif + normalsAndTangents.push_back(finalNormal); + normalsAndTangents.push_back(finalTangent); + } + + buffer->resize(mesh.vertices.size() * (sizeof(glm::vec3) + 2 * sizeof(NormalType))); + buffer->setSubData(0, mesh.vertices.size() * sizeof(glm::vec3), (const gpu::Byte*) mesh.vertices.constData()); buffer->setSubData(mesh.vertices.size() * sizeof(glm::vec3), - mesh.normals.size() * sizeof(NormalType), (const gpu::Byte*) normalsData); + mesh.normals.size() * 2 * sizeof(NormalType), (const gpu::Byte*) normalsAndTangents.data()); } _blendedVertexBuffers.push_back(buffer); } @@ -974,7 +982,7 @@ Blender::Blender(ModelPointer model, int blendNumber, const Geometry::WeakPointe void Blender::run() { DETAILED_PROFILE_RANGE_EX(simulation_animation, __FUNCTION__, 0xFFFF0000, 0, { { "url", _model->getURL().toString() } }); - QVector vertices, normals; + QVector vertices, normals, tangents; if (_model) { int offset = 0; foreach (const FBXMesh& mesh, _meshes) { @@ -983,8 +991,10 @@ void Blender::run() { } vertices += mesh.vertices; normals += mesh.normals; + tangents += mesh.tangents; glm::vec3* meshVertices = vertices.data() + offset; glm::vec3* meshNormals = normals.data() + offset; + glm::vec3* meshTangents = tangents.data() + offset; offset += mesh.vertices.size(); const float NORMAL_COEFFICIENT_SCALE = 0.01f; for (int i = 0, n = qMin(_blendshapeCoefficients.size(), mesh.blendshapes.size()); i < n; i++) { @@ -999,6 +1009,7 @@ void Blender::run() { int index = blendshape.indices.at(j); meshVertices[index] += blendshape.vertices.at(j) * vertexCoefficient; meshNormals[index] += blendshape.normals.at(j) * normalCoefficient; + meshTangents[index] += blendshape.tangents.at(j) * normalCoefficient; } } } @@ -1007,7 +1018,7 @@ void Blender::run() { QMetaObject::invokeMethod(DependencyManager::get().data(), "setBlendedVertices", Q_ARG(ModelPointer, _model), Q_ARG(int, _blendNumber), Q_ARG(const Geometry::WeakPointer&, _geometry), Q_ARG(const QVector&, vertices), - Q_ARG(const QVector&, normals)); + Q_ARG(const QVector&, normals), Q_ARG(const QVector&, tangents)); } void Model::setScaleToFit(bool scaleToFit, const glm::vec3& dimensions, bool forceRescale) { @@ -1188,7 +1199,7 @@ bool Model::maybeStartBlender() { } void Model::setBlendedVertices(int blendNumber, const Geometry::WeakPointer& geometry, - const QVector& vertices, const QVector& normals) { + const QVector& vertices, const QVector& normals, const QVector& tangents) { auto geometryRef = geometry.lock(); if (!geometryRef || _renderGeometry != geometryRef || _blendedVertexBuffers.empty() || blendNumber < _appliedBlendNumber) { return; @@ -1196,9 +1207,7 @@ void Model::setBlendedVertices(int blendNumber, const Geometry::WeakPointer& geo _appliedBlendNumber = blendNumber; const FBXGeometry& fbxGeometry = getFBXGeometry(); int index = 0; -#if FBX_PACK_NORMALS - std::vector packedNormals; -#endif + std::vector normalsAndTangents; for (int i = 0; i < fbxGeometry.meshes.size(); i++) { const FBXMesh& mesh = fbxGeometry.meshes.at(i); if (mesh.blendshapes.isEmpty()) { @@ -1206,22 +1215,38 @@ void Model::setBlendedVertices(int blendNumber, const Geometry::WeakPointer& geo } gpu::BufferPointer& buffer = _blendedVertexBuffers[i]; - const auto verticesSize = mesh.vertices.size() * sizeof(glm::vec3); - buffer->setSubData(0, verticesSize, (gpu::Byte*) vertices.constData() + index*sizeof(glm::vec3)); -#if FBX_PACK_NORMALS - packedNormals.clear(); - packedNormals.reserve(normals.size()); - for (auto normal : normals) { - normal = FBXReader::normalizeDirForPacking(normal); - packedNormals.push_back(glm::packSnorm3x10_1x2(glm::vec4(normal, 0.0f))); - } - const auto normalsData = packedNormals.data(); -#else - const auto normalsData = mesh.normals.constData(); -#endif - buffer->setSubData(verticesSize, normals.size() * sizeof(NormalType), (const gpu::Byte*) normalsData + index*sizeof(NormalType)); + const auto vertexCount = mesh.vertices.size(); + const auto verticesSize = vertexCount * sizeof(glm::vec3); + const auto offset = index * sizeof(glm::vec3); - index += mesh.vertices.size(); + normalsAndTangents.clear(); + normalsAndTangents.reserve(normals.size()+tangents.size()); + + // Interleave normals and tangents + auto normalsRange = std::make_pair(normals.begin() + index, normals.begin() + index + vertexCount); + auto tangentsRange = std::make_pair(tangents.begin() + index, tangents.begin() + index + vertexCount); + + for (auto normalIt = normalsRange.first, tangentIt = tangentsRange.first; + normalIt != normalsRange.second; + ++normalIt, ++tangentIt) { +#if FBX_PACK_NORMALS + const auto normal = FBXReader::normalizeDirForPacking(*normalIt); + const auto tangent = FBXReader::normalizeDirForPacking(*tangentIt); + const auto finalNormal = glm::packSnorm3x10_1x2(glm::vec4(normal, 0.0f)); + const auto finalTangent = glm::packSnorm3x10_1x2(glm::vec4(tangent, 0.0f)); +#else + const auto finalNormal = *normalIt; + const auto finalTangent = *tangentIt; +#endif + normalsAndTangents.push_back(finalNormal); + normalsAndTangents.push_back(finalTangent); + } + assert(normalsAndTangents.size() == 2 * vertexCount); + + buffer->setSubData(0, verticesSize, (gpu::Byte*) vertices.constData() + offset); + buffer->setSubData(verticesSize, 2 * vertexCount * sizeof(NormalType), (const gpu::Byte*) normalsAndTangents.data()); + + index += vertexCount; } } @@ -1388,10 +1413,11 @@ void ModelBlender::noteRequiresBlend(ModelPointer model) { } } -void ModelBlender::setBlendedVertices(ModelPointer model, int blendNumber, - const Geometry::WeakPointer& geometry, const QVector& vertices, const QVector& normals) { +void ModelBlender::setBlendedVertices(ModelPointer model, int blendNumber, const Geometry::WeakPointer& geometry, + const QVector& vertices, const QVector& normals, + const QVector& tangents) { if (model) { - model->setBlendedVertices(blendNumber, geometry, vertices, normals); + model->setBlendedVertices(blendNumber, geometry, vertices, normals, tangents); } _pendingBlenders--; { diff --git a/libraries/render-utils/src/Model.h b/libraries/render-utils/src/Model.h index 7568a17342..7019ac9b27 100644 --- a/libraries/render-utils/src/Model.h +++ b/libraries/render-utils/src/Model.h @@ -114,7 +114,7 @@ public: /// Sets blended vertices computed in a separate thread. void setBlendedVertices(int blendNumber, const Geometry::WeakPointer& geometry, - const QVector& vertices, const QVector& normals); + const QVector& vertices, const QVector& normals, const QVector& tangents); bool isLoaded() const { return (bool)_renderGeometry && _renderGeometry->isGeometryLoaded(); } bool isAddedToScene() const { return _addedToScene; } @@ -440,7 +440,7 @@ public: public slots: void setBlendedVertices(ModelPointer model, int blendNumber, const Geometry::WeakPointer& geometry, - const QVector& vertices, const QVector& normals); + const QVector& vertices, const QVector& normals, const QVector& tangents); private: using Mutex = std::mutex;