From 67218a697b461ec94f071a8fa02f80fe74161e5c Mon Sep 17 00:00:00 2001
From: David Rowe <david@ctrlaltstudio.com>
Date: Tue, 16 Jan 2018 14:01:20 +1300
Subject: [PATCH] Support FBX files that have nodes with non-XYZ rotation order

---
 libraries/fbx/src/FBXReader.cpp | 65 +++++++++++++++++++++++++++++++--
 1 file changed, 62 insertions(+), 3 deletions(-)

diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp
index 659d6dfa1e..42d25c96ea 100644
--- a/libraries/fbx/src/FBXReader.cpp
+++ b/libraries/fbx/src/FBXReader.cpp
@@ -148,6 +148,59 @@ glm::vec3 parseVec3(const QString& string) {
     return value;
 }
 
+enum RotationOrder {
+    OrderXYZ = 0,
+    OrderXZY,
+    OrderYZX,
+    OrderYXZ,
+    OrderZXY,
+    OrderZYX,
+    OrderSphericXYZ
+};
+
+bool haveReportedUnhandledRotationOrder = false; // Report error only once per FBX file.
+
+glm::vec3 convertRotationToXYZ(int rotationOrder, const glm::vec3& rotation) {
+    // Convert rotation with given rotation order to have order XYZ.
+    if (rotationOrder == OrderXYZ) {
+        return rotation;
+    }
+
+    glm::quat xyzRotation;
+
+    switch (rotationOrder) {
+        case OrderXZY:
+            xyzRotation = glm::quat(glm::radians(glm::vec3(0, rotation.y, 0)))
+                * (glm::quat(glm::radians(glm::vec3(0, 0, rotation.z))) * glm::quat(glm::radians(glm::vec3(rotation.x, 0, 0))));
+            break;
+        case OrderYZX:
+            xyzRotation = glm::quat(glm::radians(glm::vec3(rotation.x, 0, 0)))
+                * (glm::quat(glm::radians(glm::vec3(0, 0, rotation.z))) * glm::quat(glm::radians(glm::vec3(0, rotation.y, 0))));
+            break;
+        case OrderYXZ:
+            xyzRotation = glm::quat(glm::radians(glm::vec3(0, 0, rotation.z)))
+                * (glm::quat(glm::radians(glm::vec3(rotation.x, 0, 0))) * glm::quat(glm::radians(glm::vec3(0, rotation.y, 0))));
+            break;
+        case OrderZXY:
+            xyzRotation = glm::quat(glm::radians(glm::vec3(0, rotation.y, 0)))
+                * (glm::quat(glm::radians(glm::vec3(rotation.x, 0, 0))) * glm::quat(glm::radians(glm::vec3(0, 0, rotation.z))));
+            break;
+        case OrderZYX:
+            xyzRotation = glm::quat(glm::radians(glm::vec3(rotation.x, 0, 0)))
+                * (glm::quat(glm::radians(glm::vec3(0, rotation.y, 0))) * glm::quat(glm::radians(glm::vec3(0, 0, rotation.z))));
+            break;
+        default:
+            // FIXME: Handle OrderSphericXYZ.
+            if (!haveReportedUnhandledRotationOrder) {
+                qCDebug(modelformat) << "ERROR: Unhandled rotation order in FBX file:" << rotationOrder;
+                haveReportedUnhandledRotationOrder = true;
+            }
+            return rotation;
+    }
+
+    return glm::degrees(safeEulerAngles(xyzRotation));
+}
+
 QString processID(const QString& id) {
     // Blender (at least) prepends a type to the ID, so strip it out
     return id.mid(id.lastIndexOf(':') + 1);
@@ -630,6 +683,7 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS
     glm::vec3 ambientColor;
     QString hifiGlobalNodeID;
     unsigned int meshIndex = 0;
+    haveReportedUnhandledRotationOrder = false;
     foreach (const FBXNode& child, node.children) {
 
         if (child.name == "FBXHeaderExtension") {
@@ -731,6 +785,7 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS
                     glm::vec3 translation;
                     // NOTE: the euler angles as supplied by the FBX file are in degrees
                     glm::vec3 rotationOffset;
+                    int rotationOrder = OrderXYZ;  // Default rotation order set in "Definitions" node is assumed to be XYZ.
                     glm::vec3 preRotation, rotation, postRotation;
                     glm::vec3 scale = glm::vec3(1.0f, 1.0f, 1.0f);
                     glm::vec3 scalePivot, rotationPivot, scaleOffset;
@@ -764,6 +819,7 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS
                             index = 4;
                         }
                         if (properties) {
+                            static const QVariant ROTATION_ORDER = QByteArray("RotationOrder");
                             static const QVariant GEOMETRIC_TRANSLATION = QByteArray("GeometricTranslation");
                             static const QVariant GEOMETRIC_ROTATION = QByteArray("GeometricRotation");
                             static const QVariant GEOMETRIC_SCALING = QByteArray("GeometricScaling");
@@ -790,6 +846,9 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS
                                     if (childProperty == LCL_TRANSLATION) {
                                         translation = getVec3(property.properties, index);
 
+                                    } else if (childProperty == ROTATION_ORDER) {
+                                        rotationOrder = property.properties.at(index).toInt();
+
                                     } else if (childProperty == ROTATION_OFFSET) {
                                         rotationOffset = getVec3(property.properties, index);
 
@@ -797,13 +856,13 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS
                                         rotationPivot = getVec3(property.properties, index);
 
                                     } else if (childProperty == PRE_ROTATION) {
-                                        preRotation = getVec3(property.properties, index);
+                                        preRotation = convertRotationToXYZ(rotationOrder, getVec3(property.properties, index));
 
                                     } else if (childProperty == LCL_ROTATION) {
-                                        rotation = getVec3(property.properties, index);
+                                        rotation = convertRotationToXYZ(rotationOrder, getVec3(property.properties, index));
 
                                     } else if (childProperty == POST_ROTATION) {
-                                        postRotation = getVec3(property.properties, index);
+                                        postRotation = convertRotationToXYZ(rotationOrder, getVec3(property.properties, index));
 
                                     } else if (childProperty == SCALING_PIVOT) {
                                         scalePivot = getVec3(property.properties, index);