Merge pull request #12033 from Zvork/compactvb

More compact representation of meshes on GPU

libraries/entities-renderer/src/RenderablePolyVoxEntityItem.cpp

@@ -1422,27 +1422,29 @@ bool RenderablePolyVoxEntityItem::getMeshes(MeshProxyList& result) {
     }
 
     bool success = false;
-    MeshProxy* meshProxy = nullptr;
+    if (_mesh) {
-    glm::mat4 transform = voxelToLocalMatrix();
+        MeshProxy* meshProxy = nullptr;
-    withReadLock([&] {
+        glm::mat4 transform = voxelToLocalMatrix();
-        gpu::BufferView::Index numVertices = (gpu::BufferView::Index)_mesh->getNumVertices();
+        withReadLock([&] {
-        if (!_meshReady) {
+            gpu::BufferView::Index numVertices = (gpu::BufferView::Index)_mesh->getNumVertices();
-            // we aren't ready to return a mesh.  the caller will have to try again later.
+            if (!_meshReady) {
-            success = false;
+                // we aren't ready to return a mesh.  the caller will have to try again later.
-        } else if (numVertices == 0) {
+                success = false;
-            // we are ready, but there are no triangles in the mesh.
+            } else if (numVertices == 0) {
-            success = true;
+                // we are ready, but there are no triangles in the mesh.
-        } else {
+                success = true;
-            success = true;
+            } else {
-            // the mesh will be in voxel-space.  transform it into object-space
+                success = true;
-            meshProxy = new SimpleMeshProxy(
+                // the mesh will be in voxel-space.  transform it into object-space
-                _mesh->map([=](glm::vec3 position){ return glm::vec3(transform * glm::vec4(position, 1.0f)); },
+                meshProxy = new SimpleMeshProxy(
-                           [=](glm::vec3 color){ return color; },
+                    _mesh->map([=](glm::vec3 position) { return glm::vec3(transform * glm::vec4(position, 1.0f)); },
-                           [=](glm::vec3 normal){ return glm::normalize(glm::vec3(transform * glm::vec4(normal, 0.0f))); },
+                    [=](glm::vec3 color) { return color; },
-                           [&](uint32_t index){ return index; }));
+                    [=](glm::vec3 normal) { return glm::normalize(glm::vec3(transform * glm::vec4(normal, 0.0f))); },
-            result << meshProxy;
+                    [&](uint32_t index) { return index; }));
-        }
+                result << meshProxy;
-    });
+            }
+        });
+    }
     return success;
 }
 

libraries/fbx/src/FBX.h

@@ -60,6 +60,7 @@ public:
     QVector<int> indices;
     QVector<glm::vec3> vertices;
     QVector<glm::vec3> normals;
+    QVector<glm::vec3> tangents;
 };
 
 struct FBXJointShapeInfo {

libraries/fbx/src/FBXReader.cpp

@@ -303,17 +303,97 @@ FBXBlendshape extractBlendshape(const FBXNode& object) {
     return blendshape;
 }
 
+using IndexAccessor = std::function<glm::vec3*(const FBXMesh&, int, int, glm::vec3*, glm::vec3&)>;
 
-void setTangents(FBXMesh& mesh, int firstIndex, int secondIndex) {
+static void setTangents(const FBXMesh& mesh, const IndexAccessor& vertexAccessor, int firstIndex, int secondIndex,
-    const glm::vec3& normal = mesh.normals.at(firstIndex);
+                        const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals, QVector<glm::vec3>& tangents) {
-    glm::vec3 bitangent = glm::cross(normal, mesh.vertices.at(secondIndex) - mesh.vertices.at(firstIndex));
+    glm::vec3 vertex[2];
-    if (glm::length(bitangent) < EPSILON) {
+    glm::vec3 normal;
-        return;
+    glm::vec3* tangent = vertexAccessor(mesh, firstIndex, secondIndex, vertex, normal);
+    if (tangent) {
+        glm::vec3 bitangent = glm::cross(normal, vertex[1] - vertex[0]);
+        if (glm::length(bitangent) < EPSILON) {
+            return;
+        }
+        glm::vec2 texCoordDelta = mesh.texCoords.at(secondIndex) - mesh.texCoords.at(firstIndex);
+        glm::vec3 normalizedNormal = glm::normalize(normal);
+        *tangent += glm::cross(glm::angleAxis(-atan2f(-texCoordDelta.t, texCoordDelta.s), normalizedNormal) *
+                                       glm::normalize(bitangent), normalizedNormal);
     }
-    glm::vec2 texCoordDelta = mesh.texCoords.at(secondIndex) - mesh.texCoords.at(firstIndex);
+}
-    glm::vec3 normalizedNormal = glm::normalize(normal);
+
-    mesh.tangents[firstIndex] += glm::cross(glm::angleAxis(-atan2f(-texCoordDelta.t, texCoordDelta.s), normalizedNormal) *
+static void createTangents(const FBXMesh& mesh, bool generateFromTexCoords,
-        glm::normalize(bitangent), normalizedNormal);
+                           const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals, QVector<glm::vec3>& tangents,
+                           IndexAccessor accessor) {
+    // if we have a normal map (and texture coordinates), we must compute tangents
+    if (generateFromTexCoords && !mesh.texCoords.isEmpty()) {
+        tangents.resize(vertices.size());
+
+        foreach(const FBXMeshPart& part, mesh.parts) {
+            for (int i = 0; i < part.quadIndices.size(); i += 4) {
+                setTangents(mesh, accessor, part.quadIndices.at(i), part.quadIndices.at(i + 1), vertices, normals, tangents);
+                setTangents(mesh, accessor, part.quadIndices.at(i + 1), part.quadIndices.at(i + 2), vertices, normals, tangents);
+                setTangents(mesh, accessor, part.quadIndices.at(i + 2), part.quadIndices.at(i + 3), vertices, normals, tangents);
+                setTangents(mesh, accessor, part.quadIndices.at(i + 3), part.quadIndices.at(i), vertices, normals, tangents);
+            }
+            // <= size - 3 in order to prevent overflowing triangleIndices when (i % 3) != 0
+            // This is most likely evidence of a further problem in extractMesh()
+            for (int i = 0; i <= part.triangleIndices.size() - 3; i += 3) {
+                setTangents(mesh, accessor, part.triangleIndices.at(i), part.triangleIndices.at(i + 1), vertices, normals, tangents);
+                setTangents(mesh, accessor, part.triangleIndices.at(i + 1), part.triangleIndices.at(i + 2), vertices, normals, tangents);
+                setTangents(mesh, accessor, part.triangleIndices.at(i + 2), part.triangleIndices.at(i), vertices, normals, tangents);
+            }
+            if ((part.triangleIndices.size() % 3) != 0) {
+                qCDebug(modelformat) << "Error in extractFBXGeometry part.triangleIndices.size() is not divisible by three ";
+            }
+        }
+    }
+}
+
+static void createMeshTangents(FBXMesh& mesh, bool generateFromTexCoords) {
+    // This is the only workaround I've found to trick the compiler into understanding that mesh.tangents isn't
+    // const in the lambda function.
+    auto& tangents = mesh.tangents;
+    createTangents(mesh, generateFromTexCoords, mesh.vertices, mesh.normals, mesh.tangents, 
+                   [&](const FBXMesh& mesh, int firstIndex, int secondIndex, glm::vec3* outVertices, glm::vec3& outNormal) {
+        outVertices[0] = mesh.vertices[firstIndex];
+        outVertices[1] = mesh.vertices[secondIndex];
+        outNormal = mesh.normals[firstIndex];
+        return &(tangents[firstIndex]);
+    });
+}
+
+static void createBlendShapeTangents(FBXMesh& mesh, bool generateFromTexCoords, FBXBlendshape& blendShape) {
+    // Create lookup to get index in blend shape from vertex index in mesh
+    std::vector<int> reverseIndices;
+    reverseIndices.resize(mesh.vertices.size());
+    std::iota(reverseIndices.begin(), reverseIndices.end(), 0);
+
+    for (int indexInBlendShape = 0; indexInBlendShape < blendShape.indices.size(); ++indexInBlendShape) {
+        auto indexInMesh = blendShape.indices[indexInBlendShape];
+        reverseIndices[indexInMesh] = indexInBlendShape;
+    }
+
+    createTangents(mesh, generateFromTexCoords, blendShape.vertices, blendShape.normals, blendShape.tangents,
+                   [&](const FBXMesh& mesh, int firstIndex, int secondIndex, glm::vec3* outVertices, glm::vec3& outNormal) {
+        const auto index1 = reverseIndices[firstIndex];
+        const auto index2 = reverseIndices[secondIndex];
+
+        if (index1 < blendShape.vertices.size()) {
+            outVertices[0] = blendShape.vertices[index1];
+            if (index2 < blendShape.vertices.size()) {
+                outVertices[1] = blendShape.vertices[index2];
+            } else {
+                // Index isn't in the blend shape so return vertex from mesh
+                outVertices[1] = mesh.vertices[secondIndex];
+            }
+            outNormal = blendShape.normals[index1];
+            return &blendShape.tangents[index1];
+        } else {
+            // Index isn't in blend shape so return nullptr
+            return (glm::vec3*)nullptr;
+        }
+    });
 }
 
 QVector<int> getIndices(const QVector<QString> ids, QVector<QString> modelIDs) {
@@ -1570,27 +1650,9 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS
             }
         }
 
-        // if we have a normal map (and texture coordinates), we must compute tangents
+        createMeshTangents(extracted.mesh, generateTangents);
-        if (generateTangents && !extracted.mesh.texCoords.isEmpty()) {
+        for (auto& blendShape : extracted.mesh.blendshapes) {
-            extracted.mesh.tangents.resize(extracted.mesh.vertices.size());
+            createBlendShapeTangents(extracted.mesh, generateTangents, blendShape);
-            foreach (const FBXMeshPart& part, extracted.mesh.parts) {
-                for (int i = 0; i < part.quadIndices.size(); i += 4) {
-                    setTangents(extracted.mesh, part.quadIndices.at(i), part.quadIndices.at(i + 1));
-                    setTangents(extracted.mesh, part.quadIndices.at(i + 1), part.quadIndices.at(i + 2));
-                    setTangents(extracted.mesh, part.quadIndices.at(i + 2), part.quadIndices.at(i + 3));
-                    setTangents(extracted.mesh, part.quadIndices.at(i + 3), part.quadIndices.at(i));
-                }
-                // <= size - 3 in order to prevent overflowing triangleIndices when (i % 3) != 0
-                // This is most likely evidence of a further problem in extractMesh()
-                for (int i = 0; i <= part.triangleIndices.size() - 3; i += 3) {
-                    setTangents(extracted.mesh, part.triangleIndices.at(i), part.triangleIndices.at(i + 1));
-                    setTangents(extracted.mesh, part.triangleIndices.at(i + 1), part.triangleIndices.at(i + 2));
-                    setTangents(extracted.mesh, part.triangleIndices.at(i + 2), part.triangleIndices.at(i));
-                }
-                if ((part.triangleIndices.size() % 3) != 0){
-                    qCDebug(modelformat) << "Error in extractFBXGeometry part.triangleIndices.size() is not divisible by three ";
-                }
-            }
         }
 
         // find the clusters with which the mesh is associated

libraries/fbx/src/FBXReader.h

@@ -33,6 +33,15 @@
 class QIODevice;
 class FBXNode;
 
+#define FBX_PACK_NORMALS 1
+
+#if FBX_PACK_NORMALS
+using NormalType = glm::uint32;
+#define FBX_NORMAL_ELEMENT gpu::Element::VEC4F_NORMALIZED_XYZ10W2
+#else
+using NormalType = glm::vec3;
+#define FBX_NORMAL_ELEMENT gpu::Element::VEC3F_XYZ
+#endif
 
 /// Reads FBX geometry from the supplied model and mapping data.
 /// \exception QString if an error occurs in parsing
@@ -114,6 +123,8 @@ public:
     QHash<QString, ExtractedMesh> meshes;
     static void buildModelMesh(FBXMesh& extractedMesh, const QString& url);
 
+    static glm::vec3 normalizeDirForPacking(const glm::vec3& dir);
+
     FBXTexture getTexture(const QString& textureID);
 
     QHash<QString, QString> _textureNames;

libraries/fbx/src/FBXReader_Mesh.cpp

@@ -37,6 +37,20 @@
 
 #include <memory>
 
+#include <glm/detail/type_half.hpp>
+#include <glm/gtc/packing.hpp>
+
+using vec2h = glm::tvec2<glm::detail::hdata>;
+
+#define FBX_PACK_COLORS 1
+
+#if FBX_PACK_COLORS
+using ColorType = glm::uint32;
+#define FBX_COLOR_ELEMENT gpu::Element::COLOR_RGBA_32
+#else
+using ColorType = glm::vec3;
+#define FBX_COLOR_ELEMENT gpu::Element::VEC3F_XYZ
+#endif
 
 class Vertex {
 public:
@@ -225,7 +239,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn
             foreach (const FBXNode& subdata, child.children) {
                 if (subdata.name == "Colors") {
                     data.colors = createVec4VectorRGBA(getDoubleVector(subdata), data.averageColor);
-                } else if (subdata.name == "ColorsIndex") {
+                } else if (subdata.name == "ColorsIndex" || subdata.name == "ColorIndex") {
                     data.colorIndices = getIntVector(subdata);
 
                 } else if (subdata.name == "MappingInformationType" && subdata.properties.at(0) == BY_VERTICE) {
@@ -543,6 +557,14 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn
     return data.extracted;
 }
 
+glm::vec3 FBXReader::normalizeDirForPacking(const glm::vec3& dir) {
+    auto maxCoord = glm::max(fabsf(dir.x), glm::max(fabsf(dir.y), fabsf(dir.z)));
+    if (maxCoord > 1e-6f) {
+        return dir / maxCoord;
+    }
+    return dir;
+}
+
 void FBXReader::buildModelMesh(FBXMesh& extractedMesh, const QString& url) {
     static QString repeatedMessage = LogHandler::getInstance().addRepeatedMessageRegex("buildModelMesh failed -- .*");
 
@@ -571,37 +593,115 @@ void FBXReader::buildModelMesh(FBXMesh& extractedMesh, const QString& url) {
     gpu::BufferView vbv(vb, gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ));
     mesh->setVertexBuffer(vbv);
 
+    if (!fbxMesh.normals.empty() && fbxMesh.tangents.empty()) {
+        // Fill with a dummy value to force tangents to be present if there are normals
+        fbxMesh.tangents.reserve(fbxMesh.normals.size());
+        std::fill_n(std::back_inserter(fbxMesh.tangents), fbxMesh.normals.size(), Vectors::UNIT_X);
+    }
+    // Same thing with blend shapes
+    for (auto& blendShape : fbxMesh.blendshapes) {
+        if (!blendShape.normals.empty() && blendShape.tangents.empty()) {
+            // Fill with a dummy value to force tangents to be present if there are normals
+            blendShape.tangents.reserve(blendShape.normals.size());
+            std::fill_n(std::back_inserter(fbxMesh.tangents), blendShape.normals.size(), Vectors::UNIT_X);
+        }
+    }
+
     // evaluate all attribute channels sizes
-    int normalsSize = fbxMesh.normals.size() * sizeof(glm::vec3);
+    const int normalsSize = fbxMesh.normals.size() * sizeof(NormalType);
-    int tangentsSize = fbxMesh.tangents.size() * sizeof(glm::vec3);
+    const int tangentsSize = fbxMesh.tangents.size() * sizeof(NormalType);
-    int colorsSize = fbxMesh.colors.size() * sizeof(glm::vec3);
+    // If there are normals then there should be tangents
-    int texCoordsSize = fbxMesh.texCoords.size() * sizeof(glm::vec2);
+    assert(normalsSize == tangentsSize);
-    int texCoords1Size = fbxMesh.texCoords1.size() * sizeof(glm::vec2);
+    const auto normalsAndTangentsSize = normalsSize + tangentsSize;
+    const int normalsAndTangentsStride = 2 * sizeof(NormalType);
+    const int colorsSize = fbxMesh.colors.size() * sizeof(ColorType);
+    // Texture coordinates are stored in 2 half floats
+    const int texCoordsSize = fbxMesh.texCoords.size() * sizeof(vec2h);
+    const int texCoords1Size = fbxMesh.texCoords1.size() * sizeof(vec2h);
 
     int clusterIndicesSize = fbxMesh.clusterIndices.size() * sizeof(uint8_t);
     if (fbxMesh.clusters.size() > UINT8_MAX) {
         // we need 16 bits instead of just 8 for clusterIndices
         clusterIndicesSize *= 2;
     }
-    int clusterWeightsSize = fbxMesh.clusterWeights.size() * sizeof(uint8_t);
+    const int clusterWeightsSize = fbxMesh.clusterWeights.size() * sizeof(uint8_t);
 
-    int normalsOffset = 0;
+    // Normals and tangents are interleaved
-    int tangentsOffset = normalsOffset + normalsSize;
+    const int normalsOffset = 0;
-    int colorsOffset = tangentsOffset + tangentsSize;
+    const int tangentsOffset = normalsOffset + sizeof(NormalType);
-    int texCoordsOffset = colorsOffset + colorsSize;
+    const int colorsOffset = normalsOffset + normalsSize + tangentsSize;
-    int texCoords1Offset = texCoordsOffset + texCoordsSize;
+    const int texCoordsOffset = colorsOffset + colorsSize;
-    int clusterIndicesOffset = texCoords1Offset + texCoords1Size;
+    const int texCoords1Offset = texCoordsOffset + texCoordsSize;
-    int clusterWeightsOffset = clusterIndicesOffset + clusterIndicesSize;
+    const int clusterIndicesOffset = texCoords1Offset + texCoords1Size;
-    int totalAttributeSize = clusterWeightsOffset + clusterWeightsSize;
+    const int clusterWeightsOffset = clusterIndicesOffset + clusterIndicesSize;
+    const int totalAttributeSize = clusterWeightsOffset + clusterWeightsSize;
 
     // Copy all attribute data in a single attribute buffer
     auto attribBuffer = std::make_shared<gpu::Buffer>();
     attribBuffer->resize(totalAttributeSize);
-    attribBuffer->setSubData(normalsOffset, normalsSize, (gpu::Byte*) fbxMesh.normals.constData());
+
-    attribBuffer->setSubData(tangentsOffset, tangentsSize, (gpu::Byte*) fbxMesh.tangents.constData());
+    // Interleave normals and tangents
-    attribBuffer->setSubData(colorsOffset, colorsSize, (gpu::Byte*) fbxMesh.colors.constData());
+    if (normalsSize > 0) {
-    attribBuffer->setSubData(texCoordsOffset, texCoordsSize, (gpu::Byte*) fbxMesh.texCoords.constData());
+        std::vector<NormalType> normalsAndTangents;
-    attribBuffer->setSubData(texCoords1Offset, texCoords1Size, (gpu::Byte*) fbxMesh.texCoords1.constData());
+
+        normalsAndTangents.reserve(fbxMesh.normals.size() + fbxMesh.tangents.size());
+        for (auto normalIt = fbxMesh.normals.constBegin(), tangentIt = fbxMesh.tangents.constBegin();
+             normalIt != fbxMesh.normals.constEnd();
+             ++normalIt, ++tangentIt) {
+#if FBX_PACK_NORMALS
+            const auto normal = normalizeDirForPacking(*normalIt);
+            const auto tangent = normalizeDirForPacking(*tangentIt);
+            const auto packedNormal = glm::packSnorm3x10_1x2(glm::vec4(normal, 0.0f));
+            const auto packedTangent = glm::packSnorm3x10_1x2(glm::vec4(tangent, 0.0f));
+#else
+            const auto packedNormal = *normalIt;
+            const auto packedTangent = *tangentIt;
+#endif
+            normalsAndTangents.push_back(packedNormal);
+            normalsAndTangents.push_back(packedTangent);
+        }
+        attribBuffer->setSubData(normalsOffset, normalsAndTangentsSize, (const gpu::Byte*) normalsAndTangents.data());
+    }
+
+    if (colorsSize > 0) {
+#if FBX_PACK_COLORS
+        std::vector<ColorType> colors;
+
+        colors.reserve(fbxMesh.colors.size());
+        for (const auto& color : fbxMesh.colors) {
+            colors.push_back(glm::packUnorm4x8(glm::vec4(color, 1.0f)));
+        }
+        attribBuffer->setSubData(colorsOffset, colorsSize, (const gpu::Byte*) colors.data());
+#else
+        attribBuffer->setSubData(colorsOffset, colorsSize, (const gpu::Byte*) fbxMesh.colors.constData());
+#endif
+    }
+
+    if (texCoordsSize > 0) {
+        QVector<vec2h> texCoordData;
+        texCoordData.reserve(fbxMesh.texCoords.size());
+        for (auto& texCoordVec2f : fbxMesh.texCoords) {
+            vec2h texCoordVec2h;
+
+            texCoordVec2h.x = glm::detail::toFloat16(texCoordVec2f.x);
+            texCoordVec2h.y = glm::detail::toFloat16(texCoordVec2f.y);
+            texCoordData.push_back(texCoordVec2h);
+        }
+        attribBuffer->setSubData(texCoordsOffset, texCoordsSize, (const gpu::Byte*) texCoordData.constData());
+    }
+
+    if (texCoords1Size > 0) {
+        QVector<vec2h> texCoordData;
+        texCoordData.reserve(fbxMesh.texCoords1.size());
+        for (auto& texCoordVec2f : fbxMesh.texCoords1) {
+            vec2h texCoordVec2h;
+
+            texCoordVec2h.x = glm::detail::toFloat16(texCoordVec2f.x);
+            texCoordVec2h.y = glm::detail::toFloat16(texCoordVec2f.y);
+            texCoordData.push_back(texCoordVec2h);
+        }
+        attribBuffer->setSubData(texCoords1Offset, texCoords1Size, (const gpu::Byte*) texCoordData.constData());
+    }
 
     if (fbxMesh.clusters.size() < UINT8_MAX) {
         // yay! we can fit the clusterIndices within 8-bits
@@ -612,40 +712,37 @@ void FBXReader::buildModelMesh(FBXMesh& extractedMesh, const QString& url) {
             assert(fbxMesh.clusterIndices[i] <= UINT8_MAX);
             clusterIndices[i] = (uint8_t)(fbxMesh.clusterIndices[i]);
         }
-        attribBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (gpu::Byte*) clusterIndices.constData());
+        attribBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) clusterIndices.constData());
     } else {
-        attribBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (gpu::Byte*) fbxMesh.clusterIndices.constData());
+        attribBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) fbxMesh.clusterIndices.constData());
     }
-    attribBuffer->setSubData(clusterWeightsOffset, clusterWeightsSize, (gpu::Byte*) fbxMesh.clusterWeights.constData());
+    attribBuffer->setSubData(clusterWeightsOffset, clusterWeightsSize, (const gpu::Byte*) fbxMesh.clusterWeights.constData());
 
     if (normalsSize) {
         mesh->addAttribute(gpu::Stream::NORMAL,
-                            model::BufferView(attribBuffer, normalsOffset, normalsSize,
+                           model::BufferView(attribBuffer, normalsOffset, normalsAndTangentsSize,
-                            gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ)));
+                           normalsAndTangentsStride, FBX_NORMAL_ELEMENT));
-    }
-    if (tangentsSize) {
         mesh->addAttribute(gpu::Stream::TANGENT,
-                            model::BufferView(attribBuffer, tangentsOffset, tangentsSize,
+                           model::BufferView(attribBuffer, tangentsOffset, normalsAndTangentsSize,
-                            gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ)));
+                           normalsAndTangentsStride, FBX_NORMAL_ELEMENT));
     }
     if (colorsSize) {
         mesh->addAttribute(gpu::Stream::COLOR,
-                            model::BufferView(attribBuffer, colorsOffset, colorsSize,
+                           model::BufferView(attribBuffer, colorsOffset, colorsSize, FBX_COLOR_ELEMENT));
-                            gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::RGB)));
     }
     if (texCoordsSize) {
         mesh->addAttribute(gpu::Stream::TEXCOORD,
-                            model::BufferView( attribBuffer, texCoordsOffset, texCoordsSize,
+                           model::BufferView( attribBuffer, texCoordsOffset, texCoordsSize,
-                            gpu::Element(gpu::VEC2, gpu::FLOAT, gpu::UV)));
+                           gpu::Element(gpu::VEC2, gpu::HALF, gpu::UV)));
     }
     if (texCoords1Size) {
         mesh->addAttribute( gpu::Stream::TEXCOORD1,
                             model::BufferView(attribBuffer, texCoords1Offset, texCoords1Size,
-                            gpu::Element(gpu::VEC2, gpu::FLOAT, gpu::UV)));
+                            gpu::Element(gpu::VEC2, gpu::HALF, gpu::UV)));
     } else if (texCoordsSize) {
         mesh->addAttribute(gpu::Stream::TEXCOORD1,
-                            model::BufferView(attribBuffer, texCoordsOffset, texCoordsSize,
+                           model::BufferView(attribBuffer, texCoordsOffset, texCoordsSize,
-                            gpu::Element(gpu::VEC2, gpu::FLOAT, gpu::UV)));
+                           gpu::Element(gpu::VEC2, gpu::HALF, gpu::UV)));
     }
 
     if (clusterIndicesSize) {

libraries/gpu-gl/src/gpu/gl/GLShared.h

@@ -110,7 +110,8 @@ static const GLenum ELEMENT_TYPE_TO_GL[gpu::NUM_TYPES] = {
     GL_UNSIGNED_SHORT,
     GL_BYTE,
     GL_UNSIGNED_BYTE,
-    GL_UNSIGNED_BYTE
+    GL_UNSIGNED_BYTE,
+    GL_INT_2_10_10_10_REV,
 };
 
 bool checkGLError(const char* name = nullptr);

libraries/gpu-gl/src/gpu/gl/GLTexelFormat.cpp

@@ -218,6 +218,7 @@ GLenum GLTexelFormat::evalGLTexelFormatInternal(const gpu::Element& dstFormat) {
                         case gpu::NINT8:
                             result = GL_RGBA8_SNORM;
                             break;
+                        case gpu::NINT2_10_10_10:
                         case gpu::NUINT32:
                         case gpu::NINT32:
                         case gpu::COMPRESSED:
@@ -502,6 +503,7 @@ GLTexelFormat GLTexelFormat::evalGLTexelFormat(const Element& dstFormat, const E
                 }
                 case gpu::COMPRESSED:
                 case gpu::NUINT2:
+                case gpu::NINT2_10_10_10:
                 case gpu::NUM_TYPES: { // quiet compiler
                     Q_UNREACHABLE();
                 }
@@ -553,6 +555,7 @@ GLTexelFormat GLTexelFormat::evalGLTexelFormat(const Element& dstFormat, const E
                 }
                 case gpu::COMPRESSED:
                 case gpu::NUINT2:
+                case gpu::NINT2_10_10_10:
                 case gpu::NUM_TYPES: { // quiet compiler
                     Q_UNREACHABLE();
                 }
@@ -671,6 +674,7 @@ GLTexelFormat GLTexelFormat::evalGLTexelFormat(const Element& dstFormat, const E
                     break;
                 case gpu::NUINT32:
                 case gpu::NINT32:
+                case gpu::NINT2_10_10_10:
                 case gpu::COMPRESSED:
                 case gpu::NUM_TYPES: // quiet compiler
                     Q_UNREACHABLE();

libraries/gpu/src/gpu/Format.cpp

@@ -38,6 +38,7 @@ const Element Element::VEC2F_UV{ VEC2, FLOAT, UV };
 const Element Element::VEC2F_XY{ VEC2, FLOAT, XY };
 const Element Element::VEC3F_XYZ{ VEC3, FLOAT, XYZ };
 const Element Element::VEC4F_XYZW{ VEC4, FLOAT, XYZW };
+const Element Element::VEC4F_NORMALIZED_XYZ10W2{ VEC4, NINT2_10_10_10, XYZW };
 const Element Element::INDEX_UINT16 { SCALAR, UINT16, INDEX };
 const Element Element::INDEX_INT32 { SCALAR, INT32, INDEX };
 const Element Element::PART_DRAWCALL{ VEC4, UINT32, PART };

libraries/gpu/src/gpu/Format.h

@@ -39,6 +39,7 @@ enum Type : uint8_t {
     NINT8,
     NUINT8,
     NUINT2,
+    NINT2_10_10_10,
 
     COMPRESSED,
 
@@ -65,6 +66,7 @@ static const int TYPE_SIZE[NUM_TYPES] = {
     2,
     1,
     1,
+    4,
 
     1
 };
@@ -86,6 +88,7 @@ static const bool TYPE_IS_INTEGER[NUM_TYPES] = {
     false,
     false,
     false,
+    false,
 
     false,
 };
@@ -326,6 +329,7 @@ public:
     static const Element VEC2F_XY;
     static const Element VEC3F_XYZ;
     static const Element VEC4F_XYZW;
+    static const Element VEC4F_NORMALIZED_XYZ10W2;
     static const Element INDEX_UINT16;
     static const Element INDEX_INT32;
     static const Element PART_DRAWCALL;

libraries/gpu/src/gpu/Stream.cpp

@@ -92,6 +92,15 @@ bool Stream::Format::setAttribute(Slot slot, Slot channel, Frequency frequency)
     return true;
 }
 
+Stream::Attribute Stream::Format::getAttribute(Slot slot) const {
+    auto attribIt = _attributes.find(slot);
+    if (attribIt != _attributes.end()) {
+        return attribIt->second;
+    } else {
+        return Attribute();
+    }
+}
+
 void BufferStream::addBuffer(const BufferPointer& buffer, Offset offset, Offset stride) {
     _buffers.push_back(buffer);
     _offsets.push_back(offset);

libraries/gpu/src/gpu/Stream.h

@@ -112,6 +112,7 @@ public:
         bool setAttribute(Slot slot, Slot channel, Frequency frequency = PER_VERTEX);
 
         bool hasAttribute(Slot slot) const { return (_attributes.find(slot) != _attributes.end()); }
+        Attribute getAttribute(Slot slot) const;
 
         const std::string& getKey() const { return _key; }
 

libraries/model/src/model/Geometry.cpp

@@ -11,6 +11,8 @@
 
 #include "Geometry.h"
 
+#include <glm/gtc/packing.hpp>
+
 using namespace model;
 
 Mesh::Mesh() :
@@ -72,12 +74,14 @@ void Mesh::evalVertexStream() {
 
     int channelNum = 0;
     if (hasVertexData()) {
-        _vertexStream.addBuffer(_vertexBuffer._buffer, _vertexBuffer._offset, _vertexFormat->getChannelStride(channelNum));
+        auto stride = glm::max<gpu::Offset>(_vertexFormat->getChannelStride(channelNum), _vertexBuffer._stride);
+        _vertexStream.addBuffer(_vertexBuffer._buffer, _vertexBuffer._offset, stride);
         channelNum++;
     }
     for (auto attrib : _attributeBuffers) {
         BufferView& view = attrib.second;
-        _vertexStream.addBuffer(view._buffer, view._offset, _vertexFormat->getChannelStride(channelNum));
+        auto stride = glm::max<gpu::Offset>(_vertexFormat->getChannelStride(channelNum), view._stride);
+        _vertexStream.addBuffer(view._buffer, view._offset, stride);
         channelNum++;
     }
 }
@@ -137,13 +141,15 @@ model::MeshPointer Mesh::map(std::function<glm::vec3(glm::vec3)> vertexFunc,
                              std::function<glm::vec3(glm::vec3)> colorFunc,
                              std::function<glm::vec3(glm::vec3)> normalFunc,
                              std::function<uint32_t(uint32_t)> indexFunc) const {
+    const auto vertexFormat = getVertexFormat();
+
     // vertex data
     const gpu::BufferView& vertexBufferView = getVertexBuffer();
     gpu::BufferView::Index numVertices = (gpu::BufferView::Index)getNumVertices();
 
     gpu::Resource::Size vertexSize = numVertices * sizeof(glm::vec3);
-    unsigned char* resultVertexData = new unsigned char[vertexSize];
+    std::unique_ptr<unsigned char> resultVertexData{ new unsigned char[vertexSize] };
-    unsigned char* vertexDataCursor = resultVertexData;
+    unsigned char* vertexDataCursor = resultVertexData.get();
 
     for (gpu::BufferView::Index i = 0; i < numVertices; i++) {
         glm::vec3 pos = vertexFunc(vertexBufferView.get<glm::vec3>(i));
@@ -157,13 +163,24 @@ model::MeshPointer Mesh::map(std::function<glm::vec3(glm::vec3)> vertexFunc,
     gpu::BufferView::Index numColors = (gpu::BufferView::Index)colorsBufferView.getNumElements();
 
     gpu::Resource::Size colorSize = numColors * sizeof(glm::vec3);
-    unsigned char* resultColorData = new unsigned char[colorSize];
+    std::unique_ptr<unsigned char> resultColorData{ new unsigned char[colorSize] };
-    unsigned char* colorDataCursor = resultColorData;
+    unsigned char* colorDataCursor = resultColorData.get();
+    auto colorAttribute = vertexFormat->getAttribute(attributeTypeColor);
 
-    for (gpu::BufferView::Index i = 0; i < numColors; i++) {
+    if (colorAttribute._element == gpu::Element::VEC3F_XYZ) {
-        glm::vec3 color = colorFunc(colorsBufferView.get<glm::vec3>(i));
+        for (gpu::BufferView::Index i = 0; i < numColors; i++) {
-        memcpy(colorDataCursor, &color, sizeof(color));
+            glm::vec3 color = colorFunc(colorsBufferView.get<glm::vec3>(i));
-        colorDataCursor += sizeof(color);
+            memcpy(colorDataCursor, &color, sizeof(color));
+            colorDataCursor += sizeof(color);
+        }
+    } else if (colorAttribute._element == gpu::Element::COLOR_RGBA_32) {
+        for (gpu::BufferView::Index i = 0; i < numColors; i++) {
+            auto rawColor = colorsBufferView.get<glm::uint32>(i);
+            auto color = glm::vec3(glm::unpackUnorm4x8(rawColor));
+            color = colorFunc(color);
+            memcpy(colorDataCursor, &color, sizeof(color));
+            colorDataCursor += sizeof(color);
+        }
     }
 
     // normal data
@@ -171,22 +188,34 @@ model::MeshPointer Mesh::map(std::function<glm::vec3(glm::vec3)> vertexFunc,
     const gpu::BufferView& normalsBufferView = getAttributeBuffer(attributeTypeNormal);
     gpu::BufferView::Index numNormals = (gpu::BufferView::Index)normalsBufferView.getNumElements();
     gpu::Resource::Size normalSize = numNormals * sizeof(glm::vec3);
-    unsigned char* resultNormalData = new unsigned char[normalSize];
+    std::unique_ptr<unsigned char> resultNormalData{ new unsigned char[normalSize] };
-    unsigned char* normalDataCursor = resultNormalData;
+    unsigned char* normalDataCursor = resultNormalData.get();
+    auto normalAttribute = vertexFormat->getAttribute(attributeTypeNormal);
 
-    for (gpu::BufferView::Index i = 0; i < numNormals; i++) {
+    if (normalAttribute._element == gpu::Element::VEC3F_XYZ) {
-        glm::vec3 normal = normalFunc(normalsBufferView.get<glm::vec3>(i));
+        for (gpu::BufferView::Index i = 0; i < numNormals; i++) {
-        memcpy(normalDataCursor, &normal, sizeof(normal));
+            glm::vec3 normal = normalFunc(normalsBufferView.get<glm::vec3>(i));
-        normalDataCursor += sizeof(normal);
+            memcpy(normalDataCursor, &normal, sizeof(normal));
+            normalDataCursor += sizeof(normal);
+        }
+    } else if (normalAttribute._element == gpu::Element::VEC4F_NORMALIZED_XYZ10W2) {
+        for (gpu::BufferView::Index i = 0; i < numColors; i++) {
+            auto packedNormal = normalsBufferView.get<glm::uint32>(i);
+            auto normal = glm::vec3(glm::unpackSnorm3x10_1x2(packedNormal));
+            normal = normalFunc(normal);
+            memcpy(normalDataCursor, &normal, sizeof(normal));
+            normalDataCursor += sizeof(normal);
+        }
     }
+
     // TODO -- other attributes
 
     // face data
     const gpu::BufferView& indexBufferView = getIndexBuffer();
     gpu::BufferView::Index numIndexes = (gpu::BufferView::Index)getNumIndices();
     gpu::Resource::Size indexSize = numIndexes * sizeof(uint32_t);
-    unsigned char* resultIndexData = new unsigned char[indexSize];
+    std::unique_ptr<unsigned char> resultIndexData{ new unsigned char[indexSize] };
-    unsigned char* indexDataCursor = resultIndexData;
+    unsigned char* indexDataCursor = resultIndexData.get();
 
     for (gpu::BufferView::Index i = 0; i < numIndexes; i++) {
         uint32_t index = indexFunc(indexBufferView.get<uint32_t>(i));
@@ -197,25 +226,25 @@ model::MeshPointer Mesh::map(std::function<glm::vec3(glm::vec3)> vertexFunc,
     model::MeshPointer result(new model::Mesh());
 
     gpu::Element vertexElement = gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ);
-    gpu::Buffer* resultVertexBuffer = new gpu::Buffer(vertexSize, resultVertexData);
+    gpu::Buffer* resultVertexBuffer = new gpu::Buffer(vertexSize, resultVertexData.get());
     gpu::BufferPointer resultVertexBufferPointer(resultVertexBuffer);
     gpu::BufferView resultVertexBufferView(resultVertexBufferPointer, vertexElement);
     result->setVertexBuffer(resultVertexBufferView);
 
     gpu::Element colorElement = gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ);
-    gpu::Buffer* resultColorsBuffer = new gpu::Buffer(colorSize, resultColorData);
+    gpu::Buffer* resultColorsBuffer = new gpu::Buffer(colorSize, resultColorData.get());
     gpu::BufferPointer resultColorsBufferPointer(resultColorsBuffer);
     gpu::BufferView resultColorsBufferView(resultColorsBufferPointer, colorElement);
     result->addAttribute(attributeTypeColor, resultColorsBufferView);
 
     gpu::Element normalElement = gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ);
-    gpu::Buffer* resultNormalsBuffer = new gpu::Buffer(normalSize, resultNormalData);
+    gpu::Buffer* resultNormalsBuffer = new gpu::Buffer(normalSize, resultNormalData.get());
     gpu::BufferPointer resultNormalsBufferPointer(resultNormalsBuffer);
     gpu::BufferView resultNormalsBufferView(resultNormalsBufferPointer, normalElement);
     result->addAttribute(attributeTypeNormal, resultNormalsBufferView);
 
     gpu::Element indexElement = gpu::Element(gpu::SCALAR, gpu::UINT32, gpu::RAW);
-    gpu::Buffer* resultIndexesBuffer = new gpu::Buffer(indexSize, resultIndexData);
+    gpu::Buffer* resultIndexesBuffer = new gpu::Buffer(indexSize, resultIndexData.get());
     gpu::BufferPointer resultIndexesBufferPointer(resultIndexesBuffer);
     gpu::BufferView resultIndexesBufferView(resultIndexesBufferPointer, indexElement);
     result->setIndexBuffer(resultIndexesBufferView);
@@ -239,6 +268,8 @@ void Mesh::forEach(std::function<void(glm::vec3)> vertexFunc,
                    std::function<void(glm::vec3)> colorFunc,
                    std::function<void(glm::vec3)> normalFunc,
                    std::function<void(uint32_t)> indexFunc) {
+    const auto vertexFormat = getVertexFormat();
+
     // vertex data
     const gpu::BufferView& vertexBufferView = getVertexBuffer();
     gpu::BufferView::Index numVertices = (gpu::BufferView::Index)getNumVertices();
@@ -250,17 +281,36 @@ void Mesh::forEach(std::function<void(glm::vec3)> vertexFunc,
     int attributeTypeColor = gpu::Stream::InputSlot::COLOR; // libraries/gpu/src/gpu/Stream.h
     const gpu::BufferView& colorsBufferView = getAttributeBuffer(attributeTypeColor);
     gpu::BufferView::Index numColors =  (gpu::BufferView::Index)colorsBufferView.getNumElements();
-    for (gpu::BufferView::Index i = 0; i < numColors; i++) {
+    auto colorAttribute = vertexFormat->getAttribute(attributeTypeColor);
-        colorFunc(colorsBufferView.get<glm::vec3>(i));
+    if (colorAttribute._element == gpu::Element::VEC3F_XYZ) {
+        for (gpu::BufferView::Index i = 0; i < numColors; i++) {
+            colorFunc(colorsBufferView.get<glm::vec3>(i));
+        }
+    } else if (colorAttribute._element == gpu::Element::COLOR_RGBA_32) {
+        for (gpu::BufferView::Index i = 0; i < numColors; i++) {
+            auto rawColor = colorsBufferView.get<glm::uint32>(i);
+            auto color = glm::unpackUnorm4x8(rawColor);
+            colorFunc(color);
+        }
     }
 
     // normal data
     int attributeTypeNormal = gpu::Stream::InputSlot::NORMAL; // libraries/gpu/src/gpu/Stream.h
     const gpu::BufferView& normalsBufferView = getAttributeBuffer(attributeTypeNormal);
     gpu::BufferView::Index numNormals =  (gpu::BufferView::Index)normalsBufferView.getNumElements();
-    for (gpu::BufferView::Index i = 0; i < numNormals; i++) {
+    auto normalAttribute = vertexFormat->getAttribute(attributeTypeNormal);
-        normalFunc(normalsBufferView.get<glm::vec3>(i));
+    if (normalAttribute._element == gpu::Element::VEC3F_XYZ) {
+        for (gpu::BufferView::Index i = 0; i < numNormals; i++) {
+            normalFunc(normalsBufferView.get<glm::vec3>(i));
+        }
+    } else if (normalAttribute._element == gpu::Element::VEC4F_NORMALIZED_XYZ10W2) {
+        for (gpu::BufferView::Index i = 0; i < numColors; i++) {
+            auto packedNormal = normalsBufferView.get<glm::uint32>(i);
+            auto normal = glm::unpackSnorm3x10_1x2(packedNormal);
+            normalFunc(normal);
+        }
     }
+
     // TODO -- other attributes
 
     // face data

libraries/render-utils/src/MeshPartPayload.cpp

@@ -484,7 +484,8 @@ void ModelMeshPartPayload::bindMesh(gpu::Batch& batch) {
     batch.setInputFormat((_drawMesh->getVertexFormat()));
     if (_isBlendShaped && _blendedVertexBuffer) {
         batch.setInputBuffer(0, _blendedVertexBuffer, 0, sizeof(glm::vec3));
-        batch.setInputBuffer(1, _blendedVertexBuffer, _drawMesh->getNumVertices() * sizeof(glm::vec3), sizeof(glm::vec3));
+        // Stride is 2*sizeof(glm::vec3) because normal and tangents are interleaved
+        batch.setInputBuffer(1, _blendedVertexBuffer, _drawMesh->getNumVertices() * sizeof(glm::vec3), 2 * sizeof(NormalType));
         batch.setInputStream(2, _drawMesh->getVertexStream().makeRangedStream(2));
     } else {
         batch.setInputStream(0, _drawMesh->getVertexStream());

libraries/render-utils/src/Model.cpp

@@ -24,8 +24,12 @@
 #include <PerfStat.h>
 #include <ViewFrustum.h>
 #include <GLMHelpers.h>
+#include <TBBHelpers.h>
+
 #include <model-networking/SimpleMeshProxy.h>
 
+#include <glm/gtc/packing.hpp>
+
 #include "AbstractViewStateInterface.h"
 #include "MeshPartPayload.h"
 
@@ -310,6 +314,34 @@ void Model::reset() {
     }
 }
 
+#if FBX_PACK_NORMALS
+static void packNormalAndTangent(glm::vec3 normal, glm::vec3 tangent, glm::uint32& packedNormal, glm::uint32& packedTangent) {
+    auto absNormal = glm::abs(normal);
+    auto absTangent = glm::abs(tangent);
+    normal /= glm::max(1e-6f, glm::max(glm::max(absNormal.x, absNormal.y), absNormal.z));
+    tangent /= glm::max(1e-6f, glm::max(glm::max(absTangent.x, absTangent.y), absTangent.z));
+    normal = glm::clamp(normal, -1.0f, 1.0f);
+    tangent = glm::clamp(tangent, -1.0f, 1.0f);
+    normal *= 511.0f;
+    tangent *= 511.0f;
+    normal = glm::round(normal);
+    tangent = glm::round(tangent);
+
+    glm::detail::i10i10i10i2 normalStruct;
+    glm::detail::i10i10i10i2 tangentStruct;
+    normalStruct.data.x = int(normal.x);
+    normalStruct.data.y = int(normal.y);
+    normalStruct.data.z = int(normal.z);
+    normalStruct.data.w = 0;
+    tangentStruct.data.x = int(tangent.x);
+    tangentStruct.data.y = int(tangent.y);
+    tangentStruct.data.z = int(tangent.z);
+    tangentStruct.data.w = 0;
+    packedNormal = normalStruct.pack;
+    packedTangent = tangentStruct.pack;
+}
+#endif
+
 bool Model::updateGeometry() {
     bool needFullUpdate = false;
 
@@ -335,10 +367,28 @@ bool Model::updateGeometry() {
             // TODO? make _blendedVertexBuffers a map instead of vector and only add for meshes with blendshapes?
             auto buffer = std::make_shared<gpu::Buffer>();
             if (!mesh.blendshapes.isEmpty()) {
-                buffer->resize((mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3));
+                std::vector<NormalType> normalsAndTangents;
-                buffer->setSubData(0, mesh.vertices.size() * sizeof(glm::vec3), (gpu::Byte*) mesh.vertices.constData());
+                normalsAndTangents.reserve(mesh.normals.size() + mesh.tangents.size());
+
+                for (auto normalIt = mesh.normals.begin(), tangentIt = mesh.tangents.begin();
+                     normalIt != mesh.normals.end();
+                     ++normalIt, ++tangentIt) {
+#if FBX_PACK_NORMALS
+                    glm::uint32 finalNormal;
+                    glm::uint32 finalTangent;
+                    packNormalAndTangent(*normalIt, *tangentIt, finalNormal, finalTangent);
+#else
+                    const auto finalNormal = *normalIt;
+                    const auto finalTangent = *tangentIt;
+#endif
+                    normalsAndTangents.push_back(finalNormal);
+                    normalsAndTangents.push_back(finalTangent);
+                }
+
+                buffer->resize(mesh.vertices.size() * (sizeof(glm::vec3) + 2 * sizeof(NormalType)));
+                buffer->setSubData(0, mesh.vertices.size() * sizeof(glm::vec3), (const gpu::Byte*) mesh.vertices.constData());
                 buffer->setSubData(mesh.vertices.size() * sizeof(glm::vec3),
-                                   mesh.normals.size() * sizeof(glm::vec3), (gpu::Byte*) mesh.normals.constData());
+                                   mesh.normals.size() * 2 * sizeof(NormalType), (const gpu::Byte*) normalsAndTangents.data());
             }
             _blendedVertexBuffers.push_back(buffer);
         }
@@ -1006,7 +1056,7 @@ Blender::Blender(ModelPointer model, int blendNumber, const Geometry::WeakPointe
 
 void Blender::run() {
     DETAILED_PROFILE_RANGE_EX(simulation_animation, __FUNCTION__, 0xFFFF0000, 0, { { "url", _model->getURL().toString() } });
-    QVector<glm::vec3> vertices, normals;
+    QVector<glm::vec3> vertices, normals, tangents;
     if (_model) {
         int offset = 0;
         foreach (const FBXMesh& mesh, _meshes) {
@@ -1015,8 +1065,10 @@ void Blender::run() {
             }
             vertices += mesh.vertices;
             normals += mesh.normals;
+            tangents += mesh.tangents;
             glm::vec3* meshVertices = vertices.data() + offset;
             glm::vec3* meshNormals = normals.data() + offset;
+            glm::vec3* meshTangents = tangents.data() + offset;
             offset += mesh.vertices.size();
             const float NORMAL_COEFFICIENT_SCALE = 0.01f;
             for (int i = 0, n = qMin(_blendshapeCoefficients.size(), mesh.blendshapes.size()); i < n; i++) {
@@ -1031,6 +1083,7 @@ void Blender::run() {
                     int index = blendshape.indices.at(j);
                     meshVertices[index] += blendshape.vertices.at(j) * vertexCoefficient;
                     meshNormals[index] += blendshape.normals.at(j) * normalCoefficient;
+                    meshTangents[index] += blendshape.tangents.at(j) * normalCoefficient;
                 }
             }
         }
@@ -1039,7 +1092,7 @@ void Blender::run() {
     QMetaObject::invokeMethod(DependencyManager::get<ModelBlender>().data(), "setBlendedVertices",
         Q_ARG(ModelPointer, _model), Q_ARG(int, _blendNumber),
         Q_ARG(const Geometry::WeakPointer&, _geometry), Q_ARG(const QVector<glm::vec3>&, vertices),
-        Q_ARG(const QVector<glm::vec3>&, normals));
+        Q_ARG(const QVector<glm::vec3>&, normals), Q_ARG(const QVector<glm::vec3>&, tangents));
 }
 
 void Model::setScaleToFit(bool scaleToFit, const glm::vec3& dimensions, bool forceRescale) {
@@ -1220,7 +1273,7 @@ bool Model::maybeStartBlender() {
 }
 
 void Model::setBlendedVertices(int blendNumber, const Geometry::WeakPointer& geometry,
-        const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals) {
+        const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals, const QVector<glm::vec3>& tangents) {
     auto geometryRef = geometry.lock();
     if (!geometryRef || _renderGeometry != geometryRef || _blendedVertexBuffers.empty() || blendNumber < _appliedBlendNumber) {
         return;
@@ -1228,6 +1281,7 @@ void Model::setBlendedVertices(int blendNumber, const Geometry::WeakPointer& geo
     _appliedBlendNumber = blendNumber;
     const FBXGeometry& fbxGeometry = getFBXGeometry();
     int index = 0;
+    std::vector<NormalType> normalsAndTangents;
     for (int i = 0; i < fbxGeometry.meshes.size(); i++) {
         const FBXMesh& mesh = fbxGeometry.meshes.at(i);
         if (mesh.blendshapes.isEmpty()) {
@@ -1235,11 +1289,67 @@ void Model::setBlendedVertices(int blendNumber, const Geometry::WeakPointer& geo
         }
 
         gpu::BufferPointer& buffer = _blendedVertexBuffers[i];
-        buffer->setSubData(0, mesh.vertices.size() * sizeof(glm::vec3), (gpu::Byte*) vertices.constData() + index*sizeof(glm::vec3));
+        const auto vertexCount = mesh.vertices.size();
-        buffer->setSubData(mesh.vertices.size() * sizeof(glm::vec3),
+        const auto verticesSize = vertexCount * sizeof(glm::vec3);
-            mesh.normals.size() * sizeof(glm::vec3), (gpu::Byte*) normals.constData() + index*sizeof(glm::vec3));
+        const auto offset = index * sizeof(glm::vec3);
 
-        index += mesh.vertices.size();
+        normalsAndTangents.clear();
+        normalsAndTangents.resize(normals.size()+tangents.size());
+        assert(normalsAndTangents.size() == 2 * vertexCount);
+
+        // Interleave normals and tangents
+#if 0
+        // Sequential version for debugging
+        auto normalsRange = std::make_pair(normals.begin() + index, normals.begin() + index + vertexCount);
+        auto tangentsRange = std::make_pair(tangents.begin() + index, tangents.begin() + index + vertexCount);
+        auto normalsAndTangentsIt = normalsAndTangents.begin();
+
+        for (auto normalIt = normalsRange.first, tangentIt = tangentsRange.first;
+             normalIt != normalsRange.second;
+             ++normalIt, ++tangentIt) {
+#if FBX_PACK_NORMALS
+            glm::uint32 finalNormal;
+            glm::uint32 finalTangent;
+            packNormalAndTangent(*normalIt, *tangentIt, finalNormal, finalTangent);
+#else
+            const auto finalNormal = *normalIt;
+            const auto finalTangent = *tangentIt;
+#endif
+            *normalsAndTangentsIt = finalNormal;
+            ++normalsAndTangentsIt;
+            *normalsAndTangentsIt = finalTangent;
+            ++normalsAndTangentsIt;
+        }
+#else
+        // Parallel version for performance
+        tbb::parallel_for(tbb::blocked_range<size_t>(index, index+vertexCount), [&](const tbb::blocked_range<size_t>& range) {
+            auto normalsRange = std::make_pair(normals.begin() + range.begin(), normals.begin() + range.end());
+            auto tangentsRange = std::make_pair(tangents.begin() + range.begin(), tangents.begin() + range.end());
+            auto normalsAndTangentsIt = normalsAndTangents.begin() + (range.begin()-index)*2;
+
+            for (auto normalIt = normalsRange.first, tangentIt = tangentsRange.first;
+                 normalIt != normalsRange.second;
+                 ++normalIt, ++tangentIt) {
+#if FBX_PACK_NORMALS
+                glm::uint32 finalNormal;
+                glm::uint32 finalTangent;
+                packNormalAndTangent(*normalIt, *tangentIt, finalNormal, finalTangent);
+#else
+                const auto finalNormal = *normalIt;
+                const auto finalTangent = *tangentIt;
+#endif
+                *normalsAndTangentsIt = finalNormal;
+                ++normalsAndTangentsIt;
+                *normalsAndTangentsIt = finalTangent;
+                ++normalsAndTangentsIt;
+            }
+        });
+#endif
+
+        buffer->setSubData(0, verticesSize, (gpu::Byte*) vertices.constData() + offset);
+        buffer->setSubData(verticesSize, 2 * vertexCount * sizeof(NormalType), (const gpu::Byte*) normalsAndTangents.data());
+
+        index += vertexCount;
     }
 }
 
@@ -1406,10 +1516,11 @@ void ModelBlender::noteRequiresBlend(ModelPointer model) {
     }
 }
 
-void ModelBlender::setBlendedVertices(ModelPointer model, int blendNumber,
+void ModelBlender::setBlendedVertices(ModelPointer model, int blendNumber, const Geometry::WeakPointer& geometry, 
-        const Geometry::WeakPointer& geometry, const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals) {
+                                      const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals, 
+                                      const QVector<glm::vec3>& tangents) {
     if (model) {
-        model->setBlendedVertices(blendNumber, geometry, vertices, normals);
+        model->setBlendedVertices(blendNumber, geometry, vertices, normals, tangents);
     }
     _pendingBlenders--;
     {

libraries/render-utils/src/Model.h

@@ -114,7 +114,7 @@ public:
 
     /// Sets blended vertices computed in a separate thread.
     void setBlendedVertices(int blendNumber, const Geometry::WeakPointer& geometry,
-        const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals);
+        const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals, const QVector<glm::vec3>& tangents);
 
     bool isLoaded() const { return (bool)_renderGeometry && _renderGeometry->isGeometryLoaded(); }
     bool isAddedToScene() const { return _addedToScene; }
@@ -437,7 +437,7 @@ public:
 
 public slots:
     void setBlendedVertices(ModelPointer model, int blendNumber, const Geometry::WeakPointer& geometry,
-        const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals);
+        const QVector<glm::vec3>& vertices, const QVector<glm::vec3>& normals, const QVector<glm::vec3>& tangents);
 
 private:
     using Mutex = std::mutex;

libraries/script-engine/src/ModelScriptingInterface.cpp

@@ -60,20 +60,20 @@ QScriptValue ModelScriptingInterface::appendMeshes(MeshProxyList in) {
 
     // alloc the resulting mesh
     gpu::Resource::Size combinedVertexSize = totalVertexCount * sizeof(glm::vec3);
-    unsigned char* combinedVertexData  = new unsigned char[combinedVertexSize];
+    std::unique_ptr<unsigned char> combinedVertexData{ new unsigned char[combinedVertexSize] };
-    unsigned char* combinedVertexDataCursor = combinedVertexData;
+    unsigned char* combinedVertexDataCursor = combinedVertexData.get();
 
     gpu::Resource::Size combinedColorSize = totalColorCount * sizeof(glm::vec3);
-    unsigned char* combinedColorData  = new unsigned char[combinedColorSize];
+    std::unique_ptr<unsigned char> combinedColorData{ new unsigned char[combinedColorSize] };
-    unsigned char* combinedColorDataCursor = combinedColorData;
+    unsigned char* combinedColorDataCursor = combinedColorData.get();
 
     gpu::Resource::Size combinedNormalSize = totalNormalCount * sizeof(glm::vec3);
-    unsigned char* combinedNormalData  = new unsigned char[combinedNormalSize];
+    std::unique_ptr<unsigned char> combinedNormalData{ new unsigned char[combinedNormalSize] };
-    unsigned char* combinedNormalDataCursor = combinedNormalData;
+    unsigned char* combinedNormalDataCursor = combinedNormalData.get();
 
     gpu::Resource::Size combinedIndexSize = totalIndexCount * sizeof(uint32_t);
-    unsigned char* combinedIndexData  = new unsigned char[combinedIndexSize];
+    std::unique_ptr<unsigned char> combinedIndexData{ new unsigned char[combinedIndexSize] };
-    unsigned char* combinedIndexDataCursor = combinedIndexData;
+    unsigned char* combinedIndexDataCursor = combinedIndexData.get();
 
     uint32_t indexStartOffset { 0 };
 
@@ -105,27 +105,27 @@ QScriptValue ModelScriptingInterface::appendMeshes(MeshProxyList in) {
     model::MeshPointer result(new model::Mesh());
 
     gpu::Element vertexElement = gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ);
-    gpu::Buffer* combinedVertexBuffer = new gpu::Buffer(combinedVertexSize, combinedVertexData);
+    gpu::Buffer* combinedVertexBuffer = new gpu::Buffer(combinedVertexSize, combinedVertexData.get());
     gpu::BufferPointer combinedVertexBufferPointer(combinedVertexBuffer);
     gpu::BufferView combinedVertexBufferView(combinedVertexBufferPointer, vertexElement);
     result->setVertexBuffer(combinedVertexBufferView);
 
     int attributeTypeColor = gpu::Stream::InputSlot::COLOR; // libraries/gpu/src/gpu/Stream.h
     gpu::Element colorElement = gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ);
-    gpu::Buffer* combinedColorsBuffer = new gpu::Buffer(combinedColorSize, combinedColorData);
+    gpu::Buffer* combinedColorsBuffer = new gpu::Buffer(combinedColorSize, combinedColorData.get());
     gpu::BufferPointer combinedColorsBufferPointer(combinedColorsBuffer);
     gpu::BufferView combinedColorsBufferView(combinedColorsBufferPointer, colorElement);
     result->addAttribute(attributeTypeColor, combinedColorsBufferView);
 
     int attributeTypeNormal = gpu::Stream::InputSlot::NORMAL; // libraries/gpu/src/gpu/Stream.h
     gpu::Element normalElement = gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ);
-    gpu::Buffer* combinedNormalsBuffer = new gpu::Buffer(combinedNormalSize, combinedNormalData);
+    gpu::Buffer* combinedNormalsBuffer = new gpu::Buffer(combinedNormalSize, combinedNormalData.get());
     gpu::BufferPointer combinedNormalsBufferPointer(combinedNormalsBuffer);
     gpu::BufferView combinedNormalsBufferView(combinedNormalsBufferPointer, normalElement);
     result->addAttribute(attributeTypeNormal, combinedNormalsBufferView);
 
     gpu::Element indexElement = gpu::Element(gpu::SCALAR, gpu::UINT32, gpu::RAW);
-    gpu::Buffer* combinedIndexesBuffer = new gpu::Buffer(combinedIndexSize, combinedIndexData);
+    gpu::Buffer* combinedIndexesBuffer = new gpu::Buffer(combinedIndexSize, combinedIndexData.get());
     gpu::BufferPointer combinedIndexesBufferPointer(combinedIndexesBuffer);
     gpu::BufferView combinedIndexesBufferView(combinedIndexesBufferPointer, indexElement);
     result->setIndexBuffer(combinedIndexesBufferView);
@@ -152,9 +152,10 @@ QScriptValue ModelScriptingInterface::transformMesh(glm::mat4 transform, MeshPro
         return QScriptValue(false);
     }
 
+    const auto inverseTransposeTransform = glm::inverse(glm::transpose(transform));
     model::MeshPointer result = mesh->map([&](glm::vec3 position){ return glm::vec3(transform * glm::vec4(position, 1.0f)); },
                                           [&](glm::vec3 color){ return color; },
-                                          [&](glm::vec3 normal){ return glm::vec3(transform * glm::vec4(normal, 0.0f)); },
+                                          [&](glm::vec3 normal){ return glm::vec3(inverseTransposeTransform * glm::vec4(normal, 0.0f)); },
                                           [&](uint32_t index){ return index; });
     MeshProxy* resultProxy = new SimpleMeshProxy(result);
     return meshToScriptValue(_modelScriptEngine, resultProxy);