From 2383592678c8db6e4a2da782f70ea3d9232e9da0 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Thu, 12 Jan 2017 13:51:45 -0800 Subject: [PATCH 1/2] Improve FBX parsing performance by bulk loading of arrays --- libraries/fbx/src/FBXReader_Node.cpp | 69 ++++++++++++++++++---------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/libraries/fbx/src/FBXReader_Node.cpp b/libraries/fbx/src/FBXReader_Node.cpp index 85850a748e..435c4d830b 100644 --- a/libraries/fbx/src/FBXReader_Node.cpp +++ b/libraries/fbx/src/FBXReader_Node.cpp @@ -43,31 +43,54 @@ template QVariant readBinaryArray(QDataStream& in, int& position) { position += sizeof(quint32) * 3; QVector values; - const unsigned int DEFLATE_ENCODING = 1; - if (encoding == DEFLATE_ENCODING) { - // preface encoded data with uncompressed length - QByteArray compressed(sizeof(quint32) + compressedLength, 0); - *((quint32*)compressed.data()) = qToBigEndian(arrayLength * sizeof(T)); - in.readRawData(compressed.data() + sizeof(quint32), compressedLength); - position += compressedLength; - QByteArray uncompressed = qUncompress(compressed); - if (uncompressed.isEmpty()) { // answers empty byte array if corrupt - throw QString("corrupt fbx file"); - } - QDataStream uncompressedIn(uncompressed); - uncompressedIn.setByteOrder(QDataStream::LittleEndian); - uncompressedIn.setVersion(QDataStream::Qt_4_5); // for single/double precision switch - for (quint32 i = 0; i < arrayLength; i++) { - T value; - uncompressedIn >> value; - values.append(value); + if ((int)QSysInfo::ByteOrder == (int)in.byteOrder()) { + values.resize(arrayLength); + const unsigned int DEFLATE_ENCODING = 1; + QByteArray arrayData; + if (encoding == DEFLATE_ENCODING) { + // preface encoded data with uncompressed length + QByteArray compressed(sizeof(quint32) + compressedLength, 0); + *((quint32*)compressed.data()) = qToBigEndian(arrayLength * sizeof(T)); + in.readRawData(compressed.data() + sizeof(quint32), compressedLength); + position += compressedLength; + arrayData = qUncompress(compressed); + if (arrayData.isEmpty() || arrayData.size() != (sizeof(T) * arrayLength)) { // answers empty byte array if corrupt + throw QString("corrupt fbx file"); + } + } else { + arrayData.resize(sizeof(T) * arrayLength); + position += sizeof(T) * arrayLength; + in.readRawData(arrayData.data(), arrayData.size()); } + memcpy(&values[0], arrayData.constData(), arrayData.size()); } else { - for (quint32 i = 0; i < arrayLength; i++) { - T value; - in >> value; - position += streamSize(); - values.append(value); + values.reserve(arrayLength); + const unsigned int DEFLATE_ENCODING = 1; + if (encoding == DEFLATE_ENCODING) { + // preface encoded data with uncompressed length + QByteArray compressed(sizeof(quint32) + compressedLength, 0); + *((quint32*)compressed.data()) = qToBigEndian(arrayLength * sizeof(T)); + in.readRawData(compressed.data() + sizeof(quint32), compressedLength); + position += compressedLength; + QByteArray uncompressed = qUncompress(compressed); + if (uncompressed.isEmpty()) { // answers empty byte array if corrupt + throw QString("corrupt fbx file"); + } + QDataStream uncompressedIn(uncompressed); + uncompressedIn.setByteOrder(QDataStream::LittleEndian); + uncompressedIn.setVersion(QDataStream::Qt_4_5); // for single/double precision switch + for (quint32 i = 0; i < arrayLength; i++) { + T value; + uncompressedIn >> value; + values.append(value); + } + } else { + for (quint32 i = 0; i < arrayLength; i++) { + T value; + in >> value; + position += streamSize(); + values.append(value); + } } } return QVariant::fromValue(values); From 29b160d685ee5b5b38656d8d89ea4cf1ec5f38d9 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Thu, 12 Jan 2017 16:16:02 -0800 Subject: [PATCH 2/2] Reduce temp qvariant creation and qvariant conversion --- libraries/fbx/src/FBXReader.cpp | 172 ++++++++++++++++++--------- libraries/fbx/src/FBXReader_Mesh.cpp | 14 ++- 2 files changed, 123 insertions(+), 63 deletions(-) diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index 0171b95a3d..5272969e6b 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -555,8 +555,9 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS } } else if (subobject.name == "Properties70") { foreach (const FBXNode& subsubobject, subobject.children) { + static const QVariant APPLICATION_NAME = QVariant(QByteArray("Original|ApplicationName")); if (subsubobject.name == "P" && subsubobject.properties.size() >= 5 && - subsubobject.properties.at(0) == "Original|ApplicationName") { + subsubobject.properties.at(0) == APPLICATION_NAME) { geometry.applicationName = subsubobject.properties.at(4).toString(); } } @@ -571,10 +572,12 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS int index = 4; foreach (const FBXNode& subobject, object.children) { if (subobject.name == propertyName) { - QString subpropName = subobject.properties.at(0).toString(); - if (subpropName == "UnitScaleFactor") { + static const QVariant UNIT_SCALE_FACTOR = QByteArray("UnitScaleFactor"); + static const QVariant AMBIENT_COLOR = QByteArray("AmbientColor"); + const auto& subpropName = subobject.properties.at(0); + if (subpropName == UNIT_SCALE_FACTOR) { unitScaleFactor = subobject.properties.at(index).toFloat(); - } else if (subpropName == "AmbientColor") { + } else if (subpropName == AMBIENT_COLOR) { ambientColor = getVec3(subobject.properties, index); } } @@ -672,66 +675,87 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS index = 4; } if (properties) { - foreach (const FBXNode& property, subobject.children) { + static const QVariant GEOMETRIC_TRANSLATION = QByteArray("GeometricTranslation"); + static const QVariant GEOMETRIC_ROTATION = QByteArray("GeometricRotation"); + static const QVariant GEOMETRIC_SCALING = QByteArray("GeometricScaling"); + static const QVariant LCL_TRANSLATION = QByteArray("Lcl Translation"); + static const QVariant LCL_ROTATION = QByteArray("Lcl Rotation"); + static const QVariant LCL_SCALING = QByteArray("Lcl Scaling"); + static const QVariant ROTATION_MAX = QByteArray("RotationMax"); + static const QVariant ROTATION_MAX_X = QByteArray("RotationMaxX"); + static const QVariant ROTATION_MAX_Y = QByteArray("RotationMaxY"); + static const QVariant ROTATION_MAX_Z = QByteArray("RotationMaxZ"); + static const QVariant ROTATION_MIN = QByteArray("RotationMin"); + static const QVariant ROTATION_MIN_X = QByteArray("RotationMinX"); + static const QVariant ROTATION_MIN_Y = QByteArray("RotationMinY"); + static const QVariant ROTATION_MIN_Z = QByteArray("RotationMinZ"); + static const QVariant ROTATION_OFFSET = QByteArray("RotationOffset"); + static const QVariant ROTATION_PIVOT = QByteArray("RotationPivot"); + static const QVariant SCALING_OFFSET = QByteArray("ScalingOffset"); + static const QVariant SCALING_PIVOT = QByteArray("ScalingPivot"); + static const QVariant PRE_ROTATION = QByteArray("PreRotation"); + static const QVariant POST_ROTATION = QByteArray("PostRotation"); + foreach(const FBXNode& property, subobject.children) { + const auto& childProperty = property.properties.at(0); if (property.name == propertyName) { - if (property.properties.at(0) == "Lcl Translation") { + if (childProperty == LCL_TRANSLATION) { translation = getVec3(property.properties, index); - } else if (property.properties.at(0) == "RotationOffset") { + } else if (childProperty == ROTATION_OFFSET) { rotationOffset = getVec3(property.properties, index); - } else if (property.properties.at(0) == "RotationPivot") { + } else if (childProperty == ROTATION_PIVOT) { rotationPivot = getVec3(property.properties, index); - } else if (property.properties.at(0) == "PreRotation") { + } else if (childProperty == PRE_ROTATION) { preRotation = getVec3(property.properties, index); - } else if (property.properties.at(0) == "Lcl Rotation") { + } else if (childProperty == LCL_ROTATION) { rotation = getVec3(property.properties, index); - } else if (property.properties.at(0) == "PostRotation") { + } else if (childProperty == POST_ROTATION) { postRotation = getVec3(property.properties, index); - } else if (property.properties.at(0) == "ScalingPivot") { + } else if (childProperty == SCALING_PIVOT) { scalePivot = getVec3(property.properties, index); - } else if (property.properties.at(0) == "Lcl Scaling") { + } else if (childProperty == LCL_SCALING) { scale = getVec3(property.properties, index); - } else if (property.properties.at(0) == "ScalingOffset") { + } else if (childProperty == SCALING_OFFSET) { scaleOffset = getVec3(property.properties, index); // NOTE: these rotation limits are stored in degrees (NOT radians) - } else if (property.properties.at(0) == "RotationMin") { + } else if (childProperty == ROTATION_MIN) { rotationMin = getVec3(property.properties, index); - } else if (property.properties.at(0) == "RotationMax") { + } else if (childProperty == ROTATION_MAX) { rotationMax = getVec3(property.properties, index); - } else if (property.properties.at(0) == "RotationMinX") { + } else if (childProperty == ROTATION_MIN_X) { rotationMinX = property.properties.at(index).toBool(); - } else if (property.properties.at(0) == "RotationMinY") { + } else if (childProperty == ROTATION_MIN_Y) { rotationMinY = property.properties.at(index).toBool(); - } else if (property.properties.at(0) == "RotationMinZ") { + } else if (childProperty == ROTATION_MIN_Z) { rotationMinZ = property.properties.at(index).toBool(); - } else if (property.properties.at(0) == "RotationMaxX") { + } else if (childProperty == ROTATION_MAX_X) { rotationMaxX = property.properties.at(index).toBool(); - } else if (property.properties.at(0) == "RotationMaxY") { + } else if (childProperty == ROTATION_MAX_Y) { rotationMaxY = property.properties.at(index).toBool(); - } else if (property.properties.at(0) == "RotationMaxZ") { + } else if (childProperty == ROTATION_MAX_Z) { rotationMaxZ = property.properties.at(index).toBool(); - } else if (property.properties.at(0) == "GeometricTranslation") { + } else if (childProperty == GEOMETRIC_TRANSLATION) { geometricTranslation = getVec3(property.properties, index); hasGeometricOffset = true; - } else if (property.properties.at(0) == "GeometricRotation") { + } else if (childProperty == GEOMETRIC_ROTATION) { geometricRotation = getVec3(property.properties, index); hasGeometricOffset = true; - } else if (property.properties.at(0) == "GeometricScaling") { + } else if (childProperty == GEOMETRIC_SCALING) { geometricScaling = getVec3(property.properties, index); hasGeometricOffset = true; } @@ -842,20 +866,26 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS propertyName = "P"; index = 4; foreach (const FBXNode& property, subobject.children) { + static const QVariant UV_SET = QByteArray("UVSet"); + static const QVariant CURRENT_TEXTURE_BLEND_MODE = QByteArray("CurrentTextureBlendMode"); + static const QVariant USE_MATERIAL = QByteArray("UseMaterial"); + static const QVariant TRANSLATION = QByteArray("Translation"); + static const QVariant ROTATION = QByteArray("Rotation"); + static const QVariant SCALING = QByteArray("Scaling"); if (property.name == propertyName) { QString v = property.properties.at(0).toString(); - if (property.properties.at(0) == "UVSet") { + if (property.properties.at(0) == UV_SET) { std::string uvName = property.properties.at(index).toString().toStdString(); tex.assign(tex.UVSet, property.properties.at(index).toString()); - } else if (property.properties.at(0) == "CurrentTextureBlendMode") { + } else if (property.properties.at(0) == CURRENT_TEXTURE_BLEND_MODE) { tex.assign(tex.currentTextureBlendMode, property.properties.at(index).value()); - } else if (property.properties.at(0) == "UseMaterial") { + } else if (property.properties.at(0) == USE_MATERIAL) { tex.assign(tex.useMaterial, property.properties.at(index).value()); - } else if (property.properties.at(0) == "Translation") { + } else if (property.properties.at(0) == TRANSLATION) { tex.assign(tex.translation, getVec3(property.properties, index)); - } else if (property.properties.at(0) == "Rotation") { + } else if (property.properties.at(0) == ROTATION) { tex.assign(tex.rotation, getVec3(property.properties, index)); - } else if (property.properties.at(0) == "Scaling") { + } else if (property.properties.at(0) == SCALING) { tex.assign(tex.scaling, getVec3(property.properties, index)); if (tex.scaling.x == 0.0f) { tex.scaling.x = 1.0f; @@ -931,87 +961,114 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS if (properties) { std::vector unknowns; + static const QVariant DIFFUSE_COLOR = QByteArray("DiffuseColor"); + static const QVariant DIFFUSE_FACTOR = QByteArray("DiffuseFactor"); + static const QVariant DIFFUSE = QByteArray("Diffuse"); + static const QVariant SPECULAR_COLOR = QByteArray("SpecularColor"); + static const QVariant SPECULAR_FACTOR = QByteArray("SpecularFactor"); + static const QVariant SPECULAR = QByteArray("Specular"); + static const QVariant EMISSIVE_COLOR = QByteArray("EmissiveColor"); + static const QVariant EMISSIVE_FACTOR = QByteArray("EmissiveFactor"); + static const QVariant EMISSIVE = QByteArray("Emissive"); + static const QVariant AMBIENT_FACTOR = QByteArray("AmbientFactor"); + static const QVariant SHININESS = QByteArray("Shininess"); + static const QVariant OPACITY = QByteArray("Shininess"); + static const QVariant MAYA_USE_NORMAL_MAP = QByteArray("Maya|use_normal_map"); + static const QVariant MAYA_BASE_COLOR = QByteArray("Maya|base_color"); + static const QVariant MAYA_USE_COLOR_MAP = QByteArray("Maya|use_color_map"); + static const QVariant MAYA_ROUGHNESS = QByteArray("Maya|roughness"); + static const QVariant MAYA_USE_ROUGHNESS_MAP = QByteArray("Maya|use_roughness_map"); + static const QVariant MAYA_METALLIC = QByteArray("Maya|metallic"); + static const QVariant MAYA_USE_METALLIC_MAP = QByteArray("Maya|use_metallic_map"); + static const QVariant MAYA_EMISSIVE = QByteArray("Maya|emissive"); + static const QVariant MAYA_EMISSIVE_INTENSITY = QByteArray("Maya|emissive_intensity"); + static const QVariant MAYA_USE_EMISSIVE_MAP = QByteArray("Maya|use_emissive_map"); + static const QVariant MAYA_USE_AO_MAP = QByteArray("Maya|use_ao_map"); + + + + foreach(const FBXNode& property, subobject.children) { if (property.name == propertyName) { - if (property.properties.at(0) == "DiffuseColor") { + if (property.properties.at(0) == DIFFUSE_COLOR) { material.diffuseColor = getVec3(property.properties, index); - } else if (property.properties.at(0) == "DiffuseFactor") { + } else if (property.properties.at(0) == DIFFUSE_FACTOR) { material.diffuseFactor = property.properties.at(index).value(); - } else if (property.properties.at(0) == "Diffuse") { + } else if (property.properties.at(0) == DIFFUSE) { // NOTE: this is uneeded but keep it for now for debug // material.diffuseColor = getVec3(property.properties, index); // material.diffuseFactor = 1.0; - } else if (property.properties.at(0) == "SpecularColor") { + } else if (property.properties.at(0) == SPECULAR_COLOR) { material.specularColor = getVec3(property.properties, index); - } else if (property.properties.at(0) == "SpecularFactor") { + } else if (property.properties.at(0) == SPECULAR_FACTOR) { material.specularFactor = property.properties.at(index).value(); - } else if (property.properties.at(0) == "Specular") { + } else if (property.properties.at(0) == SPECULAR) { // NOTE: this is uneeded but keep it for now for debug // material.specularColor = getVec3(property.properties, index); // material.specularFactor = 1.0; - } else if (property.properties.at(0) == "EmissiveColor") { + } else if (property.properties.at(0) == EMISSIVE_COLOR) { material.emissiveColor = getVec3(property.properties, index); - } else if (property.properties.at(0) == "EmissiveFactor") { + } else if (property.properties.at(0) == EMISSIVE_FACTOR) { material.emissiveFactor = property.properties.at(index).value(); - } else if (property.properties.at(0) == "Emissive") { + } else if (property.properties.at(0) == EMISSIVE) { // NOTE: this is uneeded but keep it for now for debug // material.emissiveColor = getVec3(property.properties, index); // material.emissiveFactor = 1.0; - } else if (property.properties.at(0) == "AmbientFactor") { + } else if (property.properties.at(0) == AMBIENT_FACTOR) { material.ambientFactor = property.properties.at(index).value(); // Detected just for BLender AO vs lightmap - } else if (property.properties.at(0) == "Shininess") { + } else if (property.properties.at(0) == SHININESS) { material.shininess = property.properties.at(index).value(); - } else if (property.properties.at(0) == "Opacity") { + } else if (property.properties.at(0) == OPACITY) { material.opacity = property.properties.at(index).value(); } // Sting Ray Material Properties!!!! - else if (property.properties.at(0) == "Maya|use_normal_map") { + else if (property.properties.at(0) == MAYA_USE_NORMAL_MAP) { material.isPBSMaterial = true; material.useNormalMap = (bool)property.properties.at(index).value(); - } else if (property.properties.at(0) == "Maya|base_color") { + } else if (property.properties.at(0) == MAYA_BASE_COLOR) { material.isPBSMaterial = true; material.diffuseColor = getVec3(property.properties, index); - } else if (property.properties.at(0) == "Maya|use_color_map") { + } else if (property.properties.at(0) == MAYA_USE_COLOR_MAP) { material.isPBSMaterial = true; material.useAlbedoMap = (bool) property.properties.at(index).value(); - } else if (property.properties.at(0) == "Maya|roughness") { + } else if (property.properties.at(0) == MAYA_ROUGHNESS) { material.isPBSMaterial = true; material.roughness = property.properties.at(index).value(); - } else if (property.properties.at(0) == "Maya|use_roughness_map") { + } else if (property.properties.at(0) == MAYA_USE_ROUGHNESS_MAP) { material.isPBSMaterial = true; material.useRoughnessMap = (bool)property.properties.at(index).value(); - } else if (property.properties.at(0) == "Maya|metallic") { + } else if (property.properties.at(0) == MAYA_METALLIC) { material.isPBSMaterial = true; material.metallic = property.properties.at(index).value(); - } else if (property.properties.at(0) == "Maya|use_metallic_map") { + } else if (property.properties.at(0) == MAYA_USE_METALLIC_MAP) { material.isPBSMaterial = true; material.useMetallicMap = (bool)property.properties.at(index).value(); - } else if (property.properties.at(0) == "Maya|emissive") { + } else if (property.properties.at(0) == MAYA_EMISSIVE) { material.isPBSMaterial = true; material.emissiveColor = getVec3(property.properties, index); - } else if (property.properties.at(0) == "Maya|emissive_intensity") { + } else if (property.properties.at(0) == MAYA_EMISSIVE_INTENSITY) { material.isPBSMaterial = true; material.emissiveIntensity = property.properties.at(index).value(); - } else if (property.properties.at(0) == "Maya|use_emissive_map") { + } else if (property.properties.at(0) == MAYA_USE_EMISSIVE_MAP) { material.isPBSMaterial = true; material.useEmissiveMap = (bool)property.properties.at(index).value(); - } else if (property.properties.at(0) == "Maya|use_ao_map") { + } else if (property.properties.at(0) == MAYA_USE_AO_MAP) { material.isPBSMaterial = true; material.useOcclusionMap = (bool)property.properties.at(index).value(); @@ -1116,9 +1173,11 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS #endif } } else if (child.name == "Connections") { + static const QVariant OO = QByteArray("OO"); + static const QVariant OP = QByteArray("OP"); foreach (const FBXNode& connection, child.children) { if (connection.name == "C" || connection.name == "Connect") { - if (connection.properties.at(0) == "OO") { + if (connection.properties.at(0) == OO) { QString childID = getID(connection.properties, 1); QString parentID = getID(connection.properties, 2); ooChildToParent.insert(childID, parentID); @@ -1132,8 +1191,7 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS _lightmapOffset = glm::clamp((*lightIt).second.color.x, 0.f, 1.f); } } - } - if (connection.properties.at(0) == "OP") { + } else if (connection.properties.at(0) == OP) { int counter = 0; QByteArray type = connection.properties.at(3).toByteArray().toLower(); if (type.contains("DiffuseFactor")) { diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 5a824aa284..8eb31b0731 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -171,7 +171,8 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn QVector materials; QVector textures; bool isMaterialPerPolygon = false; - + static const QVariant BY_VERTICE = QByteArray("ByVertice"); + static const QVariant INDEX_TO_DIRECT = QByteArray("IndexToDirect"); foreach (const FBXNode& child, object.children) { if (child.name == "Vertices") { data.vertices = createVec3Vector(getDoubleVector(child)); @@ -189,10 +190,10 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn } else if (subdata.name == "NormalsIndex") { data.normalIndices = getIntVector(subdata); - } else if (subdata.name == "MappingInformationType" && subdata.properties.at(0) == "ByVertice") { + } else if (subdata.name == "MappingInformationType" && subdata.properties.at(0) == BY_VERTICE) { data.normalsByVertex = true; - } else if (subdata.name == "ReferenceInformationType" && subdata.properties.at(0) == "IndexToDirect") { + } else if (subdata.name == "ReferenceInformationType" && subdata.properties.at(0) == INDEX_TO_DIRECT) { indexToDirect = true; } } @@ -209,10 +210,10 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn } else if (subdata.name == "ColorsIndex") { data.colorIndices = getIntVector(subdata); - } else if (subdata.name == "MappingInformationType" && subdata.properties.at(0) == "ByVertice") { + } else if (subdata.name == "MappingInformationType" && subdata.properties.at(0) == BY_VERTICE) { data.colorsByVertex = true; - } else if (subdata.name == "ReferenceInformationType" && subdata.properties.at(0) == "IndexToDirect") { + } else if (subdata.name == "ReferenceInformationType" && subdata.properties.at(0) == INDEX_TO_DIRECT) { indexToDirect = true; } } @@ -298,11 +299,12 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn } } } else if (child.name == "LayerElementMaterial") { + static const QVariant BY_POLYGON = QByteArray("ByPolygon"); foreach (const FBXNode& subdata, child.children) { if (subdata.name == "Materials") { materials = getIntVector(subdata); } else if (subdata.name == "MappingInformationType") { - if (subdata.properties.at(0) == "ByPolygon") + if (subdata.properties.at(0) == BY_POLYGON) isMaterialPerPolygon = true; } else { isMaterialPerPolygon = false;