From 1706c7c97a2d73d8f7cba5d7334cb154fd5174c3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 6 Nov 2013 15:51:26 -0800 Subject: [PATCH 01/22] New IK method: will be easier to add constraints, adds "gravity" influence to sap potential energy. --- interface/src/Util.cpp | 6 ++ interface/src/Util.h | 2 + interface/src/renderer/FBXReader.cpp | 3 + interface/src/renderer/FBXReader.h | 1 + interface/src/renderer/Model.cpp | 109 +++++++++++++++++++++------ interface/src/renderer/Model.h | 12 +++ 6 files changed, 112 insertions(+), 21 deletions(-) diff --git a/interface/src/Util.cpp b/interface/src/Util.cpp index 447e1e6ccc..779ae5736b 100644 --- a/interface/src/Util.cpp +++ b/interface/src/Util.cpp @@ -162,6 +162,12 @@ glm::vec3 extractTranslation(const glm::mat4& matrix) { return glm::vec3(matrix[3][0], matrix[3][1], matrix[3][2]); } +void setTranslation(glm::mat4& matrix, const glm::vec3& translation) { + matrix[3][0] = translation.x; + matrix[3][1] = translation.y; + matrix[3][2] = translation.z; +} + glm::quat extractRotation(const glm::mat4& matrix, bool assumeOrthogonal) { // uses the iterative polar decomposition algorithm described by Ken Shoemake at // http://www.cs.wisc.edu/graphics/Courses/838-s2002/Papers/polar-decomp.pdf diff --git a/interface/src/Util.h b/interface/src/Util.h index c208b67bef..ec410429db 100644 --- a/interface/src/Util.h +++ b/interface/src/Util.h @@ -57,6 +57,8 @@ glm::quat safeMix(const glm::quat& q1, const glm::quat& q2, float alpha); glm::vec3 extractTranslation(const glm::mat4& matrix); +void setTranslation(glm::mat4& matrix, const glm::vec3& translation); + glm::quat extractRotation(const glm::mat4& matrix, bool assumeOrthogonal = false); double diffclock(timeval *clock1,timeval *clock2); diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index ccc2c2f24c..fd84b6e19c 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -1064,12 +1064,15 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) if (joint.parentIndex == -1) { joint.transform = geometry.offset * model.preTransform * glm::mat4_cast(combinedRotation) * model.postTransform; joint.inverseBindRotation = glm::inverse(combinedRotation); + joint.distanceToParent = 0.0f; } else { const FBXJoint& parentJoint = geometry.joints.at(joint.parentIndex); joint.transform = parentJoint.transform * model.preTransform * glm::mat4_cast(combinedRotation) * model.postTransform; joint.inverseBindRotation = glm::inverse(combinedRotation) * parentJoint.inverseBindRotation; + joint.distanceToParent = glm::distance(extractTranslation(parentJoint.transform), + extractTranslation(joint.transform)); } geometry.joints.append(joint); geometry.jointIndices.insert(model.name, geometry.joints.size() - 1); diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 69fec1cda9..45f92d64d3 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -46,6 +46,7 @@ public: bool isFree; QVector freeLineage; int parentIndex; + float distanceToParent; glm::mat4 preTransform; glm::quat preRotation; glm::quat rotation; diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 7df5427d91..9d4f37347c 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -485,6 +485,10 @@ bool Model::setLeftHandRotation(const glm::quat& rotation) { return isActive() && setJointRotation(_geometry->getFBXGeometry().leftHandJointIndex, rotation); } +float Model::getLeftArmLength() const { + return isActive() ? getLimbLength(_geometry->getFBXGeometry().leftHandJointIndex) : 0.0f; +} + bool Model::setRightHandPosition(const glm::vec3& position) { return isActive() && setJointPosition(_geometry->getFBXGeometry().rightHandJointIndex, position); } @@ -497,6 +501,10 @@ bool Model::setRightHandRotation(const glm::quat& rotation) { return isActive() && setJointRotation(_geometry->getFBXGeometry().rightHandJointIndex, rotation); } +float Model::getRightArmLength() const { + return isActive() ? getLimbLength(_geometry->getFBXGeometry().rightHandJointIndex) : 0.0f; +} + void Model::setURL(const QUrl& url) { // don't recreate the geometry if it's the same URL if (_url == url) { @@ -575,6 +583,28 @@ bool Model::getJointRotation(int jointIndex, glm::quat& rotation) const { return true; } +void Model::setJointTranslation(int jointIndex, int parentIndex, int childIndex, const glm::vec3& translation) { + JointState& state = _jointStates[jointIndex]; + if (childIndex != -1) { + // if there's a child, then I must adjust *my* rotation + glm::vec3 childTranslation = extractTranslation(_jointStates.at(childIndex).transform); + glm::quat delta = rotationBetween(childTranslation - extractTranslation(state.transform), + childTranslation - translation); + state.rotation = state.rotation * glm::inverse(state.combinedRotation) * delta * state.combinedRotation; + state.combinedRotation = delta * state.combinedRotation; + } + if (parentIndex != -1) { + // if there's a parent, then I must adjust *its* rotation + JointState& parent = _jointStates[parentIndex]; + glm::vec3 parentTranslation = extractTranslation(parent.transform); + glm::quat delta = rotationBetween(extractTranslation(state.transform) - parentTranslation, + translation - parentTranslation); + parent.rotation = parent.rotation * glm::inverse(parent.combinedRotation) * delta * parent.combinedRotation; + parent.combinedRotation = delta * parent.combinedRotation; + } + ::setTranslation(state.transform, translation); +} + bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { if (jointIndex == -1 || _jointStates.isEmpty()) { return false; @@ -583,35 +613,59 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { const FBXGeometry& geometry = _geometry->getFBXGeometry(); const QVector& freeLineage = geometry.joints.at(jointIndex).freeLineage; - // this is a cyclic coordinate descent algorithm: see - // http://www.ryanjuckett.com/programming/animation/21-cyclic-coordinate-descent-in-2d + // this is a constraint relaxation algorithm: see + // http://www.ryanjuckett.com/programming/animation/22-constraint-relaxation-ik-in-2d + + // start by optimistically setting the position of the end joint to our target + setJointTranslation(jointIndex, freeLineage.at(1), -1, relativePosition); + + // the influence of gravity; lowers the potential energy of our configurations + glm::vec3 gravity = _rotation * IDENTITY_UP * -0.1f; + + // over one or more iterations, apply the length constraints and update the rotations accordingly + float uniformScale = glm::length(_scale); const int ITERATION_COUNT = 1; for (int i = 0; i < ITERATION_COUNT; i++) { - // first, we go from the joint upwards, rotating the end as close as possible to the target - glm::vec3 endPosition = extractTranslation(_jointStates[jointIndex].transform); for (int j = 1; j < freeLineage.size(); j++) { - int index = freeLineage.at(j); - if (glm::distance(endPosition, relativePosition) < EPSILON) { - return true; // close enough to target position - } - const FBXJoint& joint = geometry.joints.at(index); - if (!joint.isFree) { + int sourceIndex = freeLineage.at(j); + int destIndex = freeLineage.at(j - 1); + const FBXJoint& sourceJoint = geometry.joints.at(sourceIndex); + const FBXJoint& destJoint = geometry.joints.at(destIndex); + JointState& sourceState = _jointStates[sourceIndex]; + JointState& destState = _jointStates[destIndex]; + glm::vec3 sourceTranslation = extractTranslation(sourceState.transform); + glm::vec3 destTranslation = extractTranslation(destState.transform); + glm::vec3 boneVector = destTranslation - sourceTranslation; + float boneLength = glm::length(boneVector); + if (boneLength < EPSILON) { continue; } - JointState& state = _jointStates[index]; - glm::vec3 jointPosition = extractTranslation(state.transform); - glm::vec3 jointVector = endPosition - jointPosition; - glm::quat deltaRotation = rotationBetween(jointVector, relativePosition - jointPosition); - state.rotation = state.rotation * glm::inverse(state.combinedRotation) * deltaRotation * state.combinedRotation; - endPosition = deltaRotation * jointVector + jointPosition; - } - - // then, from the first free joint downwards, update the transforms again - for (int j = freeLineage.size() - 1; j >= 0; j--) { - updateJointState(freeLineage.at(j)); + float extension = destJoint.distanceToParent * uniformScale / boneLength - 1.0f; + if (fabs(extension) < EPSILON) { + continue; + } + if (j == 1) { + setJointTranslation(sourceIndex, freeLineage.at(j + 1), -1, + sourceTranslation - boneVector * extension + gravity); + + } else if (j == freeLineage.size() - 1) { + setJointTranslation(destIndex, -1, freeLineage.at(j - 2), + destTranslation + boneVector * extension + gravity); + + } else { + setJointTranslation(sourceIndex, freeLineage.at(j + 1), -1, + sourceTranslation - boneVector * extension * 0.5f + gravity); + setJointTranslation(destIndex, -1, freeLineage.at(j - 2), + destTranslation + boneVector * extension * 0.5f + gravity); + } } } + // now update the joint states from the top + for (int i = freeLineage.size() - 1; i >= 0; i--) { + updateJointState(freeLineage.at(i)); + } + return true; } @@ -638,6 +692,19 @@ bool Model::restoreJointPosition(int jointIndex, float percent) { return true; } +float Model::getLimbLength(int jointIndex) const { + if (jointIndex == -1 || _jointStates.isEmpty()) { + return 0.0f; + } + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + const QVector& freeLineage = geometry.joints.at(jointIndex).freeLineage; + int length = 0.0f; + for (int i = freeLineage.size() - 2; i >= 0; i--) { + length += geometry.joints.at(freeLineage.at(i)).distanceToParent; + } + return length; +} + void Model::deleteGeometry() { foreach (Model* attachment, _attachments) { delete attachment; diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index f4b07f8242..0ea7ec7523 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -83,6 +83,9 @@ public: /// \return whether or not the left hand joint was found bool setLeftHandRotation(const glm::quat& rotation); + /// Returns the extended length from the left hand to its first free ancestor. + float getLeftArmLength() const; + /// Sets the position of the right hand using inverse kinematics. /// \return whether or not the right hand joint was found bool setRightHandPosition(const glm::vec3& position); @@ -96,6 +99,9 @@ public: /// \return whether or not the right hand joint was found bool setRightHandRotation(const glm::quat& rotation); + /// Returns the extended length from the right hand to its first free ancestor. + float getRightArmLength() const; + /// Returns the average color of all meshes in the geometry. glm::vec4 computeAverageColor() const; @@ -146,8 +152,14 @@ protected: /// \return true if the joint was found bool restoreJointPosition(int jointIndex, float percent = 1.0f); + /// Computes and returns the extended length of the limb terminating at the specified joint and starting at the joint's + /// first free ancestor. + float getLimbLength(int jointIndex) const; + private: + void setJointTranslation(int jointIndex, int parentIndex, int childIndex, const glm::vec3& translation); + void deleteGeometry(); float _pupilDilation; From 97cf9f2285f7184a7f958fda370e33a5a9e90c25 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 6 Nov 2013 15:58:35 -0800 Subject: [PATCH 02/22] Cleanup on aisle 3. --- interface/src/renderer/Model.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 9d4f37347c..32430f4e25 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -629,8 +629,6 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { for (int j = 1; j < freeLineage.size(); j++) { int sourceIndex = freeLineage.at(j); int destIndex = freeLineage.at(j - 1); - const FBXJoint& sourceJoint = geometry.joints.at(sourceIndex); - const FBXJoint& destJoint = geometry.joints.at(destIndex); JointState& sourceState = _jointStates[sourceIndex]; JointState& destState = _jointStates[destIndex]; glm::vec3 sourceTranslation = extractTranslation(sourceState.transform); @@ -640,7 +638,7 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { if (boneLength < EPSILON) { continue; } - float extension = destJoint.distanceToParent * uniformScale / boneLength - 1.0f; + float extension = geometry.joints.at(destIndex).distanceToParent * uniformScale / boneLength - 1.0f; if (fabs(extension) < EPSILON) { continue; } From d0cd9e979023a0803cd6ecf92fa26349681c697d Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 6 Nov 2013 16:24:25 -0800 Subject: [PATCH 03/22] Active the hand when we have leap hand data. --- interface/src/avatar/MyAvatar.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 0a95e78b91..ab711a5697 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -921,7 +921,8 @@ void MyAvatar::updateHandMovementAndTouching(float deltaTime, bool enableHandMov _avatarTouch.setHasInteractingOther(false); } - enableHandMovement |= updateLeapHandPositions(); + bool leapHands = updateLeapHandPositions(); + enableHandMovement |= leapHands; //constrain right arm length and re-adjust elbow position as it bends // NOTE - the following must be called on all avatars - not just _isMine @@ -935,7 +936,7 @@ void MyAvatar::updateHandMovementAndTouching(float deltaTime, bool enableHandMov if (_mousePressed) { _handState = HAND_STATE_GRASPING; - } else if (pointing) { + } else if (pointing || leapHands) { _handState = HAND_STATE_POINTING; } else { _handState = HAND_STATE_NULL; From 2b4db0ea0e18013eff0079a12214aa0e08c40abd Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 6 Nov 2013 16:47:03 -0800 Subject: [PATCH 04/22] Apply Leap data directly to skeleton. --- interface/src/avatar/MyAvatar.cpp | 5 ++--- interface/src/avatar/SkeletonModel.cpp | 29 +++++++++++++++++++++----- interface/src/avatar/SkeletonModel.h | 4 ++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index ab711a5697..0a95e78b91 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -921,8 +921,7 @@ void MyAvatar::updateHandMovementAndTouching(float deltaTime, bool enableHandMov _avatarTouch.setHasInteractingOther(false); } - bool leapHands = updateLeapHandPositions(); - enableHandMovement |= leapHands; + enableHandMovement |= updateLeapHandPositions(); //constrain right arm length and re-adjust elbow position as it bends // NOTE - the following must be called on all avatars - not just _isMine @@ -936,7 +935,7 @@ void MyAvatar::updateHandMovementAndTouching(float deltaTime, bool enableHandMov if (_mousePressed) { _handState = HAND_STATE_GRASPING; - } else if (pointing || leapHands) { + } else if (pointing) { _handState = HAND_STATE_POINTING; } else { _handState = HAND_STATE_NULL; diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index ffc4f1981f..4f249f791b 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -28,11 +28,26 @@ void SkeletonModel::simulate(float deltaTime) { Model::simulate(deltaTime); - if (_owningAvatar->getHandState() == HAND_STATE_NULL) { - const float HAND_RESTORATION_RATE = 0.25f; - restoreRightHandPosition(HAND_RESTORATION_RATE); - } else { - setRightHandPosition(_owningAvatar->getHandPosition()); + const float HAND_RESTORATION_RATE = 0.25f; + switch (_owningAvatar->getHand().getNumPalms()) { + case 0: // no Leap data; set hands from mouse + if (_owningAvatar->getHandState() == HAND_STATE_NULL) { + restoreRightHandPosition(HAND_RESTORATION_RATE); + } else { + setRightHandPosition(_owningAvatar->getHandPosition()); + } + restoreLeftHandPosition(HAND_RESTORATION_RATE); + break; + + case 1: // right hand only + applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, _owningAvatar->getHand().getPalms()[0]); + restoreLeftHandPosition(HAND_RESTORATION_RATE); + break; + + default: // both hands + applyPalmData(_geometry->getFBXGeometry().leftHandJointIndex, _owningAvatar->getHand().getPalms()[0]); + applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, _owningAvatar->getHand().getPalms()[1]); + break; } } @@ -86,6 +101,10 @@ bool SkeletonModel::render(float alpha) { return true; } +void SkeletonModel::applyPalmData(int jointIndex, const PalmData& palm) { + setJointPosition(jointIndex, palm.getPosition()); +} + void SkeletonModel::updateJointState(int index) { Model::updateJointState(index); diff --git a/interface/src/avatar/SkeletonModel.h b/interface/src/avatar/SkeletonModel.h index dfda39d066..d4ca88759b 100644 --- a/interface/src/avatar/SkeletonModel.h +++ b/interface/src/avatar/SkeletonModel.h @@ -9,6 +9,8 @@ #ifndef __interface__SkeletonModel__ #define __interface__SkeletonModel__ +#include + #include "renderer/Model.h" class Avatar; @@ -26,6 +28,8 @@ public: protected: + void applyPalmData(int jointIndex, const PalmData& palm); + /// Updates the state of the joint at the specified index. virtual void updateJointState(int index); From 2e9625ebd519f5d7a59539908f39c5afa45bf225 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 6 Nov 2013 16:57:30 -0800 Subject: [PATCH 05/22] Check palms' active state, apply rotations. --- interface/src/avatar/SkeletonModel.cpp | 55 +++++++++++++++++--------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 4f249f791b..3528220622 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -27,27 +27,43 @@ void SkeletonModel::simulate(float deltaTime) { setScale(glm::vec3(1.0f, 1.0f, 1.0f) * _owningAvatar->getScale() * MODEL_SCALE); Model::simulate(deltaTime); + + // find the active Leap palms, if any + int firstActivePalmIndex = -1; + int secondActivePalmIndex = -1; + for (int i = 0; i < _owningAvatar->getHand().getNumPalms(); i++) { + if (_owningAvatar->getHand().getPalms()[i].isActive()) { + if (firstActivePalmIndex == -1) { + firstActivePalmIndex = i; + } else { + secondActivePalmIndex = i; + break; + } + } + } const float HAND_RESTORATION_RATE = 0.25f; - switch (_owningAvatar->getHand().getNumPalms()) { - case 0: // no Leap data; set hands from mouse - if (_owningAvatar->getHandState() == HAND_STATE_NULL) { - restoreRightHandPosition(HAND_RESTORATION_RATE); - } else { - setRightHandPosition(_owningAvatar->getHandPosition()); - } - restoreLeftHandPosition(HAND_RESTORATION_RATE); - break; - - case 1: // right hand only - applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, _owningAvatar->getHand().getPalms()[0]); - restoreLeftHandPosition(HAND_RESTORATION_RATE); - break; - - default: // both hands - applyPalmData(_geometry->getFBXGeometry().leftHandJointIndex, _owningAvatar->getHand().getPalms()[0]); - applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, _owningAvatar->getHand().getPalms()[1]); - break; + if (firstActivePalmIndex == -1) { + // no Leap data; set hands from mouse + if (_owningAvatar->getHandState() == HAND_STATE_NULL) { + restoreRightHandPosition(HAND_RESTORATION_RATE); + } else { + setRightHandPosition(_owningAvatar->getHandPosition()); + } + restoreLeftHandPosition(HAND_RESTORATION_RATE); + + } else if (secondActivePalmIndex == -1) { + // right hand only + applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, + _owningAvatar->getHand().getPalms()[firstActivePalmIndex]); + restoreLeftHandPosition(HAND_RESTORATION_RATE); + + } else { + // both hands + applyPalmData(_geometry->getFBXGeometry().leftHandJointIndex, + _owningAvatar->getHand().getPalms()[firstActivePalmIndex]); + applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, + _owningAvatar->getHand().getPalms()[secondActivePalmIndex]); } } @@ -103,6 +119,7 @@ bool SkeletonModel::render(float alpha) { void SkeletonModel::applyPalmData(int jointIndex, const PalmData& palm) { setJointPosition(jointIndex, palm.getPosition()); + setJointRotation(jointIndex, rotationBetween(IDENTITY_UP, palm.getNormal())); } void SkeletonModel::updateJointState(int index) { From afd3f6937ce076836047970a7b7d1135b56e4ec8 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 6 Nov 2013 17:05:10 -0800 Subject: [PATCH 06/22] Derp; I don't want the length of the scale, I want the average of the components. --- interface/src/renderer/Model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 32430f4e25..b7dd65b0a0 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -623,7 +623,7 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { glm::vec3 gravity = _rotation * IDENTITY_UP * -0.1f; // over one or more iterations, apply the length constraints and update the rotations accordingly - float uniformScale = glm::length(_scale); + float uniformScale = (_scale.x + _scale.y + _scale.z) / 3.0f; const int ITERATION_COUNT = 1; for (int i = 0; i < ITERATION_COUNT; i++) { for (int j = 1; j < freeLineage.size(); j++) { From 1ae8a211bbe9baec261e801f7dbb0d86564baf4c Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 6 Nov 2013 17:34:40 -0800 Subject: [PATCH 07/22] Swap the palms if they're in the wrong order. --- interface/src/avatar/SkeletonModel.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 3528220622..55c47e7252 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -29,10 +29,11 @@ void SkeletonModel::simulate(float deltaTime) { Model::simulate(deltaTime); // find the active Leap palms, if any + HandData& hand = _owningAvatar->getHand(); int firstActivePalmIndex = -1; int secondActivePalmIndex = -1; - for (int i = 0; i < _owningAvatar->getHand().getNumPalms(); i++) { - if (_owningAvatar->getHand().getPalms()[i].isActive()) { + for (int i = 0; i < hand.getNumPalms(); i++) { + if (hand.getPalms()[i].isActive()) { if (firstActivePalmIndex == -1) { firstActivePalmIndex = i; } else { @@ -43,6 +44,7 @@ void SkeletonModel::simulate(float deltaTime) { } const float HAND_RESTORATION_RATE = 0.25f; + if (firstActivePalmIndex == -1) { // no Leap data; set hands from mouse if (_owningAvatar->getHandState() == HAND_STATE_NULL) { @@ -54,16 +56,17 @@ void SkeletonModel::simulate(float deltaTime) { } else if (secondActivePalmIndex == -1) { // right hand only - applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, - _owningAvatar->getHand().getPalms()[firstActivePalmIndex]); + applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, hand.getPalms()[firstActivePalmIndex]); restoreLeftHandPosition(HAND_RESTORATION_RATE); } else { - // both hands - applyPalmData(_geometry->getFBXGeometry().leftHandJointIndex, - _owningAvatar->getHand().getPalms()[firstActivePalmIndex]); - applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, - _owningAvatar->getHand().getPalms()[secondActivePalmIndex]); + // both hands; make sure left is first + if (hand.getPalms()[firstActivePalmIndex].getRawPosition().x > + hand.getPalms()[secondActivePalmIndex].getRawPosition().x) { + qSwap(firstActivePalmIndex, secondActivePalmIndex); + } + applyPalmData(_geometry->getFBXGeometry().leftHandJointIndex, hand.getPalms()[firstActivePalmIndex]); + applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, hand.getPalms()[secondActivePalmIndex]); } } @@ -119,7 +122,7 @@ bool SkeletonModel::render(float alpha) { void SkeletonModel::applyPalmData(int jointIndex, const PalmData& palm) { setJointPosition(jointIndex, palm.getPosition()); - setJointRotation(jointIndex, rotationBetween(IDENTITY_UP, palm.getNormal())); + setJointRotation(jointIndex, rotationBetween(_rotation * IDENTITY_UP, -palm.getNormal())); } void SkeletonModel::updateJointState(int index) { From be09d319b72b37817900fc8ad0e662905bbf0110 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 6 Nov 2013 17:58:05 -0800 Subject: [PATCH 08/22] Gravity adjustments. --- interface/src/avatar/Avatar.cpp | 7 ++++--- interface/src/renderer/Model.cpp | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 31e1ee43fb..47ac966a6d 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -660,12 +660,13 @@ void Avatar::updateArmIKAndConstraints(float deltaTime, AvatarJointID fingerTipJ float distance = glm::length(armVector); // don't let right hand get dragged beyond maximum arm length... - if (distance > _maxArmLength) { + float armLength = _maxArmLength * 0.5f; + if (distance > armLength) { // reset right hand to be constrained to maximum arm length fingerJoint.position = shoulderJoint.position; glm::vec3 armNormal = armVector / distance; - armVector = armNormal * _maxArmLength; - distance = _maxArmLength; + armVector = armNormal * armLength; + distance = armLength; glm::vec3 constrainedPosition = shoulderJoint.position; constrainedPosition += armVector; fingerJoint.position = constrainedPosition; diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index b7dd65b0a0..750b1e7676 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -616,16 +616,16 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { // this is a constraint relaxation algorithm: see // http://www.ryanjuckett.com/programming/animation/22-constraint-relaxation-ik-in-2d - // start by optimistically setting the position of the end joint to our target - setJointTranslation(jointIndex, freeLineage.at(1), -1, relativePosition); - // the influence of gravity; lowers the potential energy of our configurations - glm::vec3 gravity = _rotation * IDENTITY_UP * -0.1f; + glm::vec3 gravity = _rotation * IDENTITY_UP * -0.01f; // over one or more iterations, apply the length constraints and update the rotations accordingly float uniformScale = (_scale.x + _scale.y + _scale.z) / 3.0f; - const int ITERATION_COUNT = 1; + const int ITERATION_COUNT = 3; for (int i = 0; i < ITERATION_COUNT; i++) { + // start by optimistically setting the position of the end joint to our target + setJointTranslation(jointIndex, freeLineage.at(1), -1, relativePosition); + for (int j = 1; j < freeLineage.size(); j++) { int sourceIndex = freeLineage.at(j); int destIndex = freeLineage.at(j - 1); @@ -657,11 +657,11 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { destTranslation + boneVector * extension * 0.5f + gravity); } } - } - - // now update the joint states from the top - for (int i = freeLineage.size() - 1; i >= 0; i--) { - updateJointState(freeLineage.at(i)); + + // now update the joint states from the top + for (int j = freeLineage.size() - 1; j >= 0; j--) { + updateJointState(freeLineage.at(j)); + } } return true; From e6cd9a7368b793d1224debda490f4fd61e8a58b2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 12:12:48 -0800 Subject: [PATCH 09/22] Working on wiring up the fingers. --- interface/src/avatar/Avatar.cpp | 2 +- interface/src/avatar/SkeletonModel.cpp | 53 +++++++++++++++++++++++--- interface/src/avatar/SkeletonModel.h | 2 +- interface/src/devices/LeapManager.cpp | 4 +- interface/src/renderer/FBXReader.cpp | 26 +++++++++++++ interface/src/renderer/FBXReader.h | 3 ++ interface/src/renderer/Model.cpp | 15 +++++--- interface/src/renderer/Model.h | 2 +- 8 files changed, 92 insertions(+), 15 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 47ac966a6d..13b5cff6cb 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -660,7 +660,7 @@ void Avatar::updateArmIKAndConstraints(float deltaTime, AvatarJointID fingerTipJ float distance = glm::length(armVector); // don't let right hand get dragged beyond maximum arm length... - float armLength = _maxArmLength * 0.5f; + float armLength = _maxArmLength * 0.75f; if (distance > armLength) { // reset right hand to be constrained to maximum arm length fingerJoint.position = shoulderJoint.position; diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 55c47e7252..62dcbcac6a 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -45,6 +45,7 @@ void SkeletonModel::simulate(float deltaTime) { const float HAND_RESTORATION_RATE = 0.25f; + const FBXGeometry& geometry = _geometry->getFBXGeometry(); if (firstActivePalmIndex == -1) { // no Leap data; set hands from mouse if (_owningAvatar->getHandState() == HAND_STATE_NULL) { @@ -56,7 +57,8 @@ void SkeletonModel::simulate(float deltaTime) { } else if (secondActivePalmIndex == -1) { // right hand only - applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, hand.getPalms()[firstActivePalmIndex]); + applyPalmData(geometry.rightHandJointIndex, geometry.rightFingertipJointIndices, + hand.getPalms()[firstActivePalmIndex]); restoreLeftHandPosition(HAND_RESTORATION_RATE); } else { @@ -65,8 +67,10 @@ void SkeletonModel::simulate(float deltaTime) { hand.getPalms()[secondActivePalmIndex].getRawPosition().x) { qSwap(firstActivePalmIndex, secondActivePalmIndex); } - applyPalmData(_geometry->getFBXGeometry().leftHandJointIndex, hand.getPalms()[firstActivePalmIndex]); - applyPalmData(_geometry->getFBXGeometry().rightHandJointIndex, hand.getPalms()[secondActivePalmIndex]); + applyPalmData(geometry.leftHandJointIndex, geometry.leftFingertipJointIndices, + hand.getPalms()[firstActivePalmIndex]); + applyPalmData(geometry.rightHandJointIndex, geometry.rightFingertipJointIndices, + hand.getPalms()[secondActivePalmIndex]); } } @@ -120,9 +124,48 @@ bool SkeletonModel::render(float alpha) { return true; } -void SkeletonModel::applyPalmData(int jointIndex, const PalmData& palm) { +class IndexValue { +public: + int index; + float value; +}; + +bool operator<(const IndexValue& firstIndex, const IndexValue& secondIndex) { + return firstIndex.value < secondIndex.value; +} + +void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingertipJointIndices, PalmData& palm) { setJointPosition(jointIndex, palm.getPosition()); - setJointRotation(jointIndex, rotationBetween(_rotation * IDENTITY_UP, -palm.getNormal())); + setJointRotation(jointIndex, rotationBetween(_rotation * IDENTITY_UP, palm.getNormal()) * _rotation); + + // no point in continuing if there are no fingers + if (palm.getNumFingers() == 0) { + return; + } + + // sort the finger indices by raw x + QVector fingerIndices; + for (int i = 0; i < palm.getNumFingers(); i++) { + IndexValue indexValue = { i, palm.getFingers()[i].getTipRawPosition().x }; + fingerIndices.append(indexValue); + } + qSort(fingerIndices.begin(), fingerIndices.end()); + + // likewise with the joint indices and relative x + QVector jointIndices; + foreach (int index, fingertipJointIndices) { + glm::vec3 position = glm::inverse(_rotation) * extractTranslation(_jointStates[index].transform); + IndexValue indexValue = { index, position.x }; + jointIndices.append(indexValue); + } + //qSort(jointIndices.begin(), jointIndices.end()); + + // match them up as best we can + float proportion = fingerIndices.size() / (float)jointIndices.size(); + for (int i = 0; i < jointIndices.size(); i++) { + int fingerIndex = fingerIndices.at(roundf(i * proportion)).index; + setJointPosition(jointIndices.at(i).index, palm.getFingers()[fingerIndex].getTipPosition(), jointIndex); + } } void SkeletonModel::updateJointState(int index) { diff --git a/interface/src/avatar/SkeletonModel.h b/interface/src/avatar/SkeletonModel.h index d4ca88759b..9b32ea4929 100644 --- a/interface/src/avatar/SkeletonModel.h +++ b/interface/src/avatar/SkeletonModel.h @@ -28,7 +28,7 @@ public: protected: - void applyPalmData(int jointIndex, const PalmData& palm); + void applyPalmData(int jointIndex, const QVector& fingertipJointIndices, PalmData& palm); /// Updates the state of the joint at the specified index. virtual void updateJointState(int index); diff --git a/interface/src/devices/LeapManager.cpp b/interface/src/devices/LeapManager.cpp index 883a9c1037..07208c1fd3 100644 --- a/interface/src/devices/LeapManager.cpp +++ b/interface/src/devices/LeapManager.cpp @@ -198,8 +198,8 @@ void LeapManager::nextFrame(Avatar& avatar) { // There's no real Leap data and we need to fake it. for (size_t i = 0; i < hand.getNumPalms(); ++i) { static const glm::vec3 fakeHandOffsets[] = { - glm::vec3( -500.0f, 50.0f, 50.0f), - glm::vec3( 0.0f, 50.0f, 50.0f) + glm::vec3( -250.0f, 50.0f, 50.0f), + glm::vec3( 250.0f, 50.0f, 50.0f) }; static const glm::vec3 fakeHandFingerMirrors[] = { glm::vec3( -1.0f, 1.0f, 1.0f), diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index fd84b6e19c..5cf7a26ae9 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -747,6 +747,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QString jointHeadName = processID(joints.value("jointHead", "jointHead").toString()); QString jointLeftHandName = processID(joints.value("jointLeftHand", "jointLeftHand").toString()); QString jointRightHandName = processID(joints.value("jointRightHand", "jointRightHand").toString()); + QVariantList jointLeftFingertipNames = joints.values("jointLeftFingertip"); + QVariantList jointRightFingertipNames = joints.values("jointRightFingertip"); QString jointEyeLeftID; QString jointEyeRightID; QString jointNeckID; @@ -755,6 +757,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QString jointHeadID; QString jointLeftHandID; QString jointRightHandID; + QStringList jointLeftFingertipIDs; + QStringList jointRightFingertipIDs; QVariantHash blendshapeMappings = mapping.value("bs").toHash(); QHash > blendshapeIndices; @@ -834,6 +838,12 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } else if (name == jointRightHandName) { jointRightHandID = getID(object.properties); + + } else if (jointLeftFingertipNames.contains(name)) { + jointLeftFingertipIDs.append(getID(object.properties)); + + } else if (jointRightFingertipNames.contains(name)) { + jointRightFingertipIDs.append(getID(object.properties)); } glm::vec3 translation; glm::vec3 rotationOffset; @@ -1088,6 +1098,13 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) geometry.leftHandJointIndex = modelIDs.indexOf(jointLeftHandID); geometry.rightHandJointIndex = modelIDs.indexOf(jointRightHandID); + foreach (const QString& id, jointLeftFingertipIDs) { + geometry.leftFingertipJointIndices.append(modelIDs.indexOf(id)); + } + foreach (const QString& id, jointRightFingertipIDs) { + geometry.rightFingertipJointIndices.append(modelIDs.indexOf(id)); + } + // extract the translation component of the neck transform if (geometry.neckJointIndex != -1) { const glm::mat4& transform = geometry.joints.at(geometry.neckJointIndex).transform; @@ -1096,9 +1113,14 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QVariantHash springs = mapping.value("spring").toHash(); QVariant defaultSpring = springs.value("default"); + int vertices = 0; + int quads = 0; + int tris = 0; for (QHash::iterator it = meshes.begin(); it != meshes.end(); it++) { ExtractedMesh& extracted = it.value(); + vertices += extracted.mesh.vertices.size(); + // accumulate local transforms QString modelID = models.contains(it.key()) ? it.key() : parentMap.value(it.key()); extracted.mesh.springiness = springs.value(models.value(modelID).name, defaultSpring).toFloat(); @@ -1112,6 +1134,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) break; } FBXMeshPart& part = extracted.mesh.parts[partIndex]; + quads += part.quadIndices.size() / 4; + tris += part.triangleIndices.size() / 3; if (textureFilenames.contains(childID)) { part.diffuseFilename = textureFilenames.value(childID); continue; @@ -1272,6 +1296,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) geometry.meshes.append(extracted.mesh); } + qDebug("%d %d %d\n", vertices, quads, tris); + // process attachments QVariantHash attachments = mapping.value("attach").toHash(); for (QVariantHash::const_iterator it = attachments.constBegin(); it != attachments.constEnd(); it++) { diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 45f92d64d3..8f3ec74ae9 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -135,6 +135,9 @@ public: int leftHandJointIndex; int rightHandJointIndex; + QVector leftFingertipJointIndices; + QVector rightFingertipJointIndices; + glm::vec3 neckPivot; QVector attachments; diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 750b1e7676..304819f65b 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -584,8 +584,9 @@ bool Model::getJointRotation(int jointIndex, glm::quat& rotation) const { } void Model::setJointTranslation(int jointIndex, int parentIndex, int childIndex, const glm::vec3& translation) { + const FBXGeometry& geometry = _geometry->getFBXGeometry(); JointState& state = _jointStates[jointIndex]; - if (childIndex != -1) { + if (childIndex != -1 && geometry.joints.at(jointIndex).isFree) { // if there's a child, then I must adjust *my* rotation glm::vec3 childTranslation = extractTranslation(_jointStates.at(childIndex).transform); glm::quat delta = rotationBetween(childTranslation - extractTranslation(state.transform), @@ -593,7 +594,7 @@ void Model::setJointTranslation(int jointIndex, int parentIndex, int childIndex, state.rotation = state.rotation * glm::inverse(state.combinedRotation) * delta * state.combinedRotation; state.combinedRotation = delta * state.combinedRotation; } - if (parentIndex != -1) { + if (parentIndex != -1 && geometry.joints.at(parentIndex).isFree) { // if there's a parent, then I must adjust *its* rotation JointState& parent = _jointStates[parentIndex]; glm::vec3 parentTranslation = extractTranslation(parent.transform); @@ -605,13 +606,16 @@ void Model::setJointTranslation(int jointIndex, int parentIndex, int childIndex, ::setTranslation(state.transform, translation); } -bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { +bool Model::setJointPosition(int jointIndex, const glm::vec3& position, int lastFreeIndex) { if (jointIndex == -1 || _jointStates.isEmpty()) { return false; } glm::vec3 relativePosition = position - _translation; const FBXGeometry& geometry = _geometry->getFBXGeometry(); const QVector& freeLineage = geometry.joints.at(jointIndex).freeLineage; + if (lastFreeIndex == -1) { + lastFreeIndex = freeLineage.last(); + } // this is a constraint relaxation algorithm: see // http://www.ryanjuckett.com/programming/animation/22-constraint-relaxation-ik-in-2d @@ -626,9 +630,10 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { // start by optimistically setting the position of the end joint to our target setJointTranslation(jointIndex, freeLineage.at(1), -1, relativePosition); - for (int j = 1; j < freeLineage.size(); j++) { + for (int j = 1; freeLineage.at(j - 1) != lastFreeIndex; j++) { int sourceIndex = freeLineage.at(j); int destIndex = freeLineage.at(j - 1); + bool last = (sourceIndex == lastFreeIndex); JointState& sourceState = _jointStates[sourceIndex]; JointState& destState = _jointStates[destIndex]; glm::vec3 sourceTranslation = extractTranslation(sourceState.transform); @@ -646,7 +651,7 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position) { setJointTranslation(sourceIndex, freeLineage.at(j + 1), -1, sourceTranslation - boneVector * extension + gravity); - } else if (j == freeLineage.size() - 1) { + } else if (sourceIndex == lastFreeIndex) { setJointTranslation(destIndex, -1, freeLineage.at(j - 2), destTranslation + boneVector * extension + gravity); diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index 0ea7ec7523..e05b8e007f 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -143,7 +143,7 @@ protected: bool getJointPosition(int jointIndex, glm::vec3& position) const; bool getJointRotation(int jointIndex, glm::quat& rotation) const; - bool setJointPosition(int jointIndex, const glm::vec3& position); + bool setJointPosition(int jointIndex, const glm::vec3& position, int lastFreeIndex = -1); bool setJointRotation(int jointIndex, const glm::quat& rotation); /// Restores the indexed joint to its default position. From f0e72d8d0cf16d56b22adda5b4032cdfedad55b1 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 12:16:17 -0800 Subject: [PATCH 10/22] Removed unused variable, debugging code. --- interface/src/avatar/SkeletonModel.cpp | 2 +- interface/src/renderer/FBXReader.cpp | 9 --------- interface/src/renderer/Model.cpp | 1 - 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 62dcbcac6a..46a2b6bda6 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -149,7 +149,7 @@ void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingertipJ IndexValue indexValue = { i, palm.getFingers()[i].getTipRawPosition().x }; fingerIndices.append(indexValue); } - qSort(fingerIndices.begin(), fingerIndices.end()); + //qSort(fingerIndices.begin(), fingerIndices.end()); // likewise with the joint indices and relative x QVector jointIndices; diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 5cf7a26ae9..dce41ba18f 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -1113,14 +1113,9 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QVariantHash springs = mapping.value("spring").toHash(); QVariant defaultSpring = springs.value("default"); - int vertices = 0; - int quads = 0; - int tris = 0; for (QHash::iterator it = meshes.begin(); it != meshes.end(); it++) { ExtractedMesh& extracted = it.value(); - vertices += extracted.mesh.vertices.size(); - // accumulate local transforms QString modelID = models.contains(it.key()) ? it.key() : parentMap.value(it.key()); extracted.mesh.springiness = springs.value(models.value(modelID).name, defaultSpring).toFloat(); @@ -1134,8 +1129,6 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) break; } FBXMeshPart& part = extracted.mesh.parts[partIndex]; - quads += part.quadIndices.size() / 4; - tris += part.triangleIndices.size() / 3; if (textureFilenames.contains(childID)) { part.diffuseFilename = textureFilenames.value(childID); continue; @@ -1296,8 +1289,6 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) geometry.meshes.append(extracted.mesh); } - qDebug("%d %d %d\n", vertices, quads, tris); - // process attachments QVariantHash attachments = mapping.value("attach").toHash(); for (QVariantHash::const_iterator it = attachments.constBegin(); it != attachments.constEnd(); it++) { diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 304819f65b..8c0bd69632 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -633,7 +633,6 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position, int last for (int j = 1; freeLineage.at(j - 1) != lastFreeIndex; j++) { int sourceIndex = freeLineage.at(j); int destIndex = freeLineage.at(j - 1); - bool last = (sourceIndex == lastFreeIndex); JointState& sourceState = _jointStates[sourceIndex]; JointState& destState = _jointStates[destIndex]; glm::vec3 sourceTranslation = extractTranslation(sourceState.transform); From 4360621dafa8fe8798eb9aaf9b030ed66fdcbbb3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 13:17:08 -0800 Subject: [PATCH 11/22] Adjustments to wrist rotations. --- interface/src/avatar/SkeletonModel.cpp | 5 +++-- interface/src/renderer/FBXReader.cpp | 9 +++++++-- interface/src/renderer/FBXReader.h | 1 + interface/src/renderer/Model.cpp | 10 ++++++---- interface/src/renderer/Model.h | 4 ++-- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 46a2b6bda6..c9e9620361 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -136,10 +136,11 @@ bool operator<(const IndexValue& firstIndex, const IndexValue& secondIndex) { void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingertipJointIndices, PalmData& palm) { setJointPosition(jointIndex, palm.getPosition()); - setJointRotation(jointIndex, rotationBetween(_rotation * IDENTITY_UP, palm.getNormal()) * _rotation); + setJointRotation(jointIndex, rotationBetween(_rotation * IDENTITY_UP, palm.getNormal()) * _rotation * + glm::angleAxis(90.0f, 0.0f, jointIndex == _geometry->getFBXGeometry().rightHandJointIndex ? 1.0f : -1.0f, 0.0f), true); // no point in continuing if there are no fingers - if (palm.getNumFingers() == 0) { + if (true || palm.getNumFingers() == 0) { return; } diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index dce41ba18f..f0ef5d91f3 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -1073,17 +1073,18 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::quat combinedRotation = model.preRotation * model.rotation * model.postRotation; if (joint.parentIndex == -1) { joint.transform = geometry.offset * model.preTransform * glm::mat4_cast(combinedRotation) * model.postTransform; - joint.inverseBindRotation = glm::inverse(combinedRotation); + joint.inverseDefaultRotation = glm::inverse(combinedRotation); joint.distanceToParent = 0.0f; } else { const FBXJoint& parentJoint = geometry.joints.at(joint.parentIndex); joint.transform = parentJoint.transform * model.preTransform * glm::mat4_cast(combinedRotation) * model.postTransform; - joint.inverseBindRotation = glm::inverse(combinedRotation) * parentJoint.inverseBindRotation; + joint.inverseDefaultRotation = glm::inverse(combinedRotation) * parentJoint.inverseDefaultRotation; joint.distanceToParent = glm::distance(extractTranslation(parentJoint.transform), extractTranslation(joint.transform)); } + joint.inverseBindRotation = joint.inverseDefaultRotation; geometry.joints.append(joint); geometry.jointIndices.insert(model.name, geometry.joints.size() - 1); } @@ -1202,6 +1203,10 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } fbxCluster.inverseBindMatrix = glm::inverse(cluster.transformLink) * modelTransform; extracted.mesh.clusters.append(fbxCluster); + + // override the bind rotation with the transform link + geometry.joints[fbxCluster.jointIndex].inverseBindRotation = + glm::inverse(extractRotation(cluster.transformLink)); } } diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 8f3ec74ae9..218e1d3c27 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -53,6 +53,7 @@ public: glm::quat postRotation; glm::mat4 postTransform; glm::mat4 transform; + glm::quat inverseDefaultRotation; glm::quat inverseBindRotation; }; diff --git a/interface/src/renderer/Model.cpp b/interface/src/renderer/Model.cpp index 8c0bd69632..6eeca5cfd4 100644 --- a/interface/src/renderer/Model.cpp +++ b/interface/src/renderer/Model.cpp @@ -574,12 +574,13 @@ bool Model::getJointPosition(int jointIndex, glm::vec3& position) const { return true; } -bool Model::getJointRotation(int jointIndex, glm::quat& rotation) const { +bool Model::getJointRotation(int jointIndex, glm::quat& rotation, bool fromBind) const { if (jointIndex == -1 || _jointStates.isEmpty()) { return false; } rotation = _jointStates[jointIndex].combinedRotation * - _geometry->getFBXGeometry().joints[jointIndex].inverseBindRotation; + (fromBind ? _geometry->getFBXGeometry().joints[jointIndex].inverseBindRotation : + _geometry->getFBXGeometry().joints[jointIndex].inverseDefaultRotation); return true; } @@ -671,13 +672,14 @@ bool Model::setJointPosition(int jointIndex, const glm::vec3& position, int last return true; } -bool Model::setJointRotation(int jointIndex, const glm::quat& rotation) { +bool Model::setJointRotation(int jointIndex, const glm::quat& rotation, bool fromBind) { if (jointIndex == -1 || _jointStates.isEmpty()) { return false; } JointState& state = _jointStates[jointIndex]; state.rotation = state.rotation * glm::inverse(state.combinedRotation) * rotation * - glm::inverse(_geometry->getFBXGeometry().joints.at(jointIndex).inverseBindRotation); + glm::inverse(fromBind ? _geometry->getFBXGeometry().joints.at(jointIndex).inverseBindRotation : + _geometry->getFBXGeometry().joints.at(jointIndex).inverseDefaultRotation); return true; } diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index e05b8e007f..3a22045b6c 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -141,10 +141,10 @@ protected: virtual void maybeUpdateEyeRotation(const JointState& parentState, const FBXJoint& joint, JointState& state); bool getJointPosition(int jointIndex, glm::vec3& position) const; - bool getJointRotation(int jointIndex, glm::quat& rotation) const; + bool getJointRotation(int jointIndex, glm::quat& rotation, bool fromBind = false) const; bool setJointPosition(int jointIndex, const glm::vec3& position, int lastFreeIndex = -1); - bool setJointRotation(int jointIndex, const glm::quat& rotation); + bool setJointRotation(int jointIndex, const glm::quat& rotation, bool fromBind = false); /// Restores the indexed joint to its default position. /// \param percent the percentage of the default position to apply (i.e., 0.25f to slerp one fourth of the way to From 797645a209aed41774ef63089a01def384416c5e Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 13:29:17 -0800 Subject: [PATCH 12/22] More palm fixes. --- interface/src/avatar/SkeletonModel.cpp | 41 ++++++++++++-------------- interface/src/devices/LeapManager.cpp | 2 +- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index c9e9620361..e865d50a3f 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -28,17 +28,22 @@ void SkeletonModel::simulate(float deltaTime) { Model::simulate(deltaTime); - // find the active Leap palms, if any + // find the left and rightmost active Leap palms HandData& hand = _owningAvatar->getHand(); - int firstActivePalmIndex = -1; - int secondActivePalmIndex = -1; + int leftPalmIndex = -1; + float leftPalmX = FLT_MAX; + int rightPalmIndex = -1; + float rightPalmX = -FLT_MAX; for (int i = 0; i < hand.getNumPalms(); i++) { if (hand.getPalms()[i].isActive()) { - if (firstActivePalmIndex == -1) { - firstActivePalmIndex = i; - } else { - secondActivePalmIndex = i; - break; + float x = hand.getPalms()[i].getRawPosition().x; + if (x < leftPalmX) { + leftPalmIndex = i; + leftPalmX = x; + } + if (x > rightPalmX) { + rightPalmIndex = i; + rightPalmX = x; } } } @@ -46,7 +51,7 @@ void SkeletonModel::simulate(float deltaTime) { const float HAND_RESTORATION_RATE = 0.25f; const FBXGeometry& geometry = _geometry->getFBXGeometry(); - if (firstActivePalmIndex == -1) { + if (leftPalmIndex == -1) { // no Leap data; set hands from mouse if (_owningAvatar->getHandState() == HAND_STATE_NULL) { restoreRightHandPosition(HAND_RESTORATION_RATE); @@ -55,22 +60,14 @@ void SkeletonModel::simulate(float deltaTime) { } restoreLeftHandPosition(HAND_RESTORATION_RATE); - } else if (secondActivePalmIndex == -1) { + } else if (leftPalmIndex == rightPalmIndex) { // right hand only - applyPalmData(geometry.rightHandJointIndex, geometry.rightFingertipJointIndices, - hand.getPalms()[firstActivePalmIndex]); + applyPalmData(geometry.rightHandJointIndex, geometry.rightFingertipJointIndices, hand.getPalms()[leftPalmIndex]); restoreLeftHandPosition(HAND_RESTORATION_RATE); } else { - // both hands; make sure left is first - if (hand.getPalms()[firstActivePalmIndex].getRawPosition().x > - hand.getPalms()[secondActivePalmIndex].getRawPosition().x) { - qSwap(firstActivePalmIndex, secondActivePalmIndex); - } - applyPalmData(geometry.leftHandJointIndex, geometry.leftFingertipJointIndices, - hand.getPalms()[firstActivePalmIndex]); - applyPalmData(geometry.rightHandJointIndex, geometry.rightFingertipJointIndices, - hand.getPalms()[secondActivePalmIndex]); + applyPalmData(geometry.leftHandJointIndex, geometry.leftFingertipJointIndices, hand.getPalms()[leftPalmIndex]); + applyPalmData(geometry.rightHandJointIndex, geometry.rightFingertipJointIndices, hand.getPalms()[rightPalmIndex]); } } @@ -136,7 +133,7 @@ bool operator<(const IndexValue& firstIndex, const IndexValue& secondIndex) { void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingertipJointIndices, PalmData& palm) { setJointPosition(jointIndex, palm.getPosition()); - setJointRotation(jointIndex, rotationBetween(_rotation * IDENTITY_UP, palm.getNormal()) * _rotation * + setJointRotation(jointIndex, rotationBetween(_rotation * IDENTITY_UP, -palm.getNormal()) * _rotation * glm::angleAxis(90.0f, 0.0f, jointIndex == _geometry->getFBXGeometry().rightHandJointIndex ? 1.0f : -1.0f, 0.0f), true); // no point in continuing if there are no fingers diff --git a/interface/src/devices/LeapManager.cpp b/interface/src/devices/LeapManager.cpp index 07208c1fd3..c3dd993c16 100644 --- a/interface/src/devices/LeapManager.cpp +++ b/interface/src/devices/LeapManager.cpp @@ -218,7 +218,7 @@ void LeapManager::nextFrame(Avatar& avatar) { // Simulated data palm.setRawPosition(glm::vec3( 0.0f, 0.0f, 0.0f) + fakeHandOffsets[i]); - palm.setRawNormal(glm::vec3(0.0f, 1.0f, 0.0f)); + palm.setRawNormal(glm::vec3(0.0f, -1.0f, 0.0f)); for (size_t f = 0; f < palm.getNumFingers(); ++f) { FingerData& finger = palm.getFingers()[f]; From aec7dbca083157c56728276e30ef1b19f9d18d96 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 13:57:54 -0800 Subject: [PATCH 13/22] Attempting the fingers again. --- interface/src/avatar/SkeletonModel.cpp | 23 +++++++++++++---------- interface/src/renderer/FBXReader.cpp | 5 +++-- interface/src/renderer/FBXReader.h | 1 + 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index e865d50a3f..b66a5d9137 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -132,31 +132,34 @@ bool operator<(const IndexValue& firstIndex, const IndexValue& secondIndex) { } void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingertipJointIndices, PalmData& palm) { + const FBXGeometry& geometry = _geometry->getFBXGeometry(); setJointPosition(jointIndex, palm.getPosition()); + float sign = jointIndex == geometry.rightHandJointIndex ? 1.0f : -1.0f; setJointRotation(jointIndex, rotationBetween(_rotation * IDENTITY_UP, -palm.getNormal()) * _rotation * - glm::angleAxis(90.0f, 0.0f, jointIndex == _geometry->getFBXGeometry().rightHandJointIndex ? 1.0f : -1.0f, 0.0f), true); + glm::angleAxis(90.0f, 0.0f, sign, 0.0f), true); // no point in continuing if there are no fingers - if (true || palm.getNumFingers() == 0) { + if (palm.getNumFingers() == 0) { return; } // sort the finger indices by raw x QVector fingerIndices; - for (int i = 0; i < palm.getNumFingers(); i++) { - IndexValue indexValue = { i, palm.getFingers()[i].getTipRawPosition().x }; + for (int i = 0; i < qMin(5, (int)palm.getNumFingers()); i++) { + IndexValue indexValue = { i, palm.getFingers()[i].getRootRawPosition().x }; fingerIndices.append(indexValue); } - //qSort(fingerIndices.begin(), fingerIndices.end()); + qSort(fingerIndices.begin(), fingerIndices.end()); - // likewise with the joint indices and relative x + // likewise with the joint indices and relative z QVector jointIndices; - foreach (int index, fingertipJointIndices) { - glm::vec3 position = glm::inverse(_rotation) * extractTranslation(_jointStates[index].transform); - IndexValue indexValue = { index, position.x }; + for (int i = 0; i < qMin(5, (int)fingertipJointIndices.size()); i++) { + int index = fingertipJointIndices.at(i); + glm::vec3 position = glm::inverse(_rotation) * extractTranslation(geometry.joints.at(index).bindTransform); + IndexValue indexValue = { index, position.z * -sign }; jointIndices.append(indexValue); } - //qSort(jointIndices.begin(), jointIndices.end()); + qSort(jointIndices.begin(), jointIndices.end()); // match them up as best we can float proportion = fingerIndices.size() / (float)jointIndices.size(); diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index f0ef5d91f3..d0d80d326e 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -1205,8 +1205,9 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) extracted.mesh.clusters.append(fbxCluster); // override the bind rotation with the transform link - geometry.joints[fbxCluster.jointIndex].inverseBindRotation = - glm::inverse(extractRotation(cluster.transformLink)); + FBXJoint& joint = geometry.joints[fbxCluster.jointIndex]; + joint.inverseBindRotation = glm::inverse(extractRotation(cluster.transformLink)); + joint.bindTransform = cluster.transformLink; } } diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 218e1d3c27..962986b7f1 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -55,6 +55,7 @@ public: glm::mat4 transform; glm::quat inverseDefaultRotation; glm::quat inverseBindRotation; + glm::mat4 bindTransform; }; /// A single binding to a joint in an FBX document. From c3049d437859af8f64a336163b8443c075e3ad34 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 14:25:01 -0800 Subject: [PATCH 14/22] Another shot at the fingers. Not attempting IK on the anymore. --- interface/src/avatar/SkeletonModel.cpp | 28 ++++++++++++++------------ interface/src/avatar/SkeletonModel.h | 2 +- interface/src/renderer/FBXReader.cpp | 24 +++++++++++----------- interface/src/renderer/FBXReader.h | 4 ++-- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index b66a5d9137..660ef6ee5a 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -62,12 +62,12 @@ void SkeletonModel::simulate(float deltaTime) { } else if (leftPalmIndex == rightPalmIndex) { // right hand only - applyPalmData(geometry.rightHandJointIndex, geometry.rightFingertipJointIndices, hand.getPalms()[leftPalmIndex]); + applyPalmData(geometry.rightHandJointIndex, geometry.rightFingerJointIndices, hand.getPalms()[leftPalmIndex]); restoreLeftHandPosition(HAND_RESTORATION_RATE); } else { - applyPalmData(geometry.leftHandJointIndex, geometry.leftFingertipJointIndices, hand.getPalms()[leftPalmIndex]); - applyPalmData(geometry.rightHandJointIndex, geometry.rightFingertipJointIndices, hand.getPalms()[rightPalmIndex]); + applyPalmData(geometry.leftHandJointIndex, geometry.leftFingerJointIndices, hand.getPalms()[leftPalmIndex]); + applyPalmData(geometry.rightHandJointIndex, geometry.rightFingerJointIndices, hand.getPalms()[rightPalmIndex]); } } @@ -131,30 +131,30 @@ bool operator<(const IndexValue& firstIndex, const IndexValue& secondIndex) { return firstIndex.value < secondIndex.value; } -void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingertipJointIndices, PalmData& palm) { +void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJointIndices, PalmData& palm) { const FBXGeometry& geometry = _geometry->getFBXGeometry(); setJointPosition(jointIndex, palm.getPosition()); - float sign = jointIndex == geometry.rightHandJointIndex ? 1.0f : -1.0f; - setJointRotation(jointIndex, rotationBetween(_rotation * IDENTITY_UP, -palm.getNormal()) * _rotation * - glm::angleAxis(90.0f, 0.0f, sign, 0.0f), true); + float sign = (jointIndex == geometry.rightHandJointIndex) ? 1.0f : -1.0f; + glm::quat palmRotation = rotationBetween(_rotation * IDENTITY_UP, -palm.getNormal()) * _rotation * + glm::angleAxis(90.0f, 0.0f, sign, 0.0f); + setJointRotation(jointIndex, palmRotation, true); // no point in continuing if there are no fingers - if (palm.getNumFingers() == 0) { + if (palm.getNumFingers() == 0 || fingerJointIndices.isEmpty()) { return; } // sort the finger indices by raw x QVector fingerIndices; - for (int i = 0; i < qMin(5, (int)palm.getNumFingers()); i++) { - IndexValue indexValue = { i, palm.getFingers()[i].getRootRawPosition().x }; + for (int i = 0; i < palm.getNumFingers(); i++) { + IndexValue indexValue = { i, palm.getFingers()[i].getTipRawPosition().x }; fingerIndices.append(indexValue); } qSort(fingerIndices.begin(), fingerIndices.end()); // likewise with the joint indices and relative z QVector jointIndices; - for (int i = 0; i < qMin(5, (int)fingertipJointIndices.size()); i++) { - int index = fingertipJointIndices.at(i); + foreach (int index, fingerJointIndices) { glm::vec3 position = glm::inverse(_rotation) * extractTranslation(geometry.joints.at(index).bindTransform); IndexValue indexValue = { index, position.z * -sign }; jointIndices.append(indexValue); @@ -165,7 +165,9 @@ void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingertipJ float proportion = fingerIndices.size() / (float)jointIndices.size(); for (int i = 0; i < jointIndices.size(); i++) { int fingerIndex = fingerIndices.at(roundf(i * proportion)).index; - setJointPosition(jointIndices.at(i).index, palm.getFingers()[fingerIndex].getTipPosition(), jointIndex); + glm::vec3 fingerVector = palm.getFingers()[fingerIndex].getTipPosition() - + palm.getFingers()[fingerIndex].getRootPosition(); + setJointRotation(jointIndices.at(i).index, rotationBetween(palmRotation * glm::vec3(-sign, 0.0f, 0.0f), fingerVector) * palmRotation, true); } } diff --git a/interface/src/avatar/SkeletonModel.h b/interface/src/avatar/SkeletonModel.h index 9b32ea4929..6d6b2d148e 100644 --- a/interface/src/avatar/SkeletonModel.h +++ b/interface/src/avatar/SkeletonModel.h @@ -28,7 +28,7 @@ public: protected: - void applyPalmData(int jointIndex, const QVector& fingertipJointIndices, PalmData& palm); + void applyPalmData(int jointIndex, const QVector& fingerJointIndices, PalmData& palm); /// Updates the state of the joint at the specified index. virtual void updateJointState(int index); diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index d0d80d326e..07b2b1a04e 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -747,8 +747,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QString jointHeadName = processID(joints.value("jointHead", "jointHead").toString()); QString jointLeftHandName = processID(joints.value("jointLeftHand", "jointLeftHand").toString()); QString jointRightHandName = processID(joints.value("jointRightHand", "jointRightHand").toString()); - QVariantList jointLeftFingertipNames = joints.values("jointLeftFingertip"); - QVariantList jointRightFingertipNames = joints.values("jointRightFingertip"); + QVariantList jointLeftFingerNames = joints.values("jointLeftFinger"); + QVariantList jointRightFingerNames = joints.values("jointRightFinger"); QString jointEyeLeftID; QString jointEyeRightID; QString jointNeckID; @@ -757,8 +757,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QString jointHeadID; QString jointLeftHandID; QString jointRightHandID; - QStringList jointLeftFingertipIDs; - QStringList jointRightFingertipIDs; + QStringList jointLeftFingerIDs; + QStringList jointRightFingerIDs; QVariantHash blendshapeMappings = mapping.value("bs").toHash(); QHash > blendshapeIndices; @@ -839,11 +839,11 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } else if (name == jointRightHandName) { jointRightHandID = getID(object.properties); - } else if (jointLeftFingertipNames.contains(name)) { - jointLeftFingertipIDs.append(getID(object.properties)); + } else if (jointLeftFingerNames.contains(name)) { + jointLeftFingerIDs.append(getID(object.properties)); - } else if (jointRightFingertipNames.contains(name)) { - jointRightFingertipIDs.append(getID(object.properties)); + } else if (jointRightFingerNames.contains(name)) { + jointRightFingerIDs.append(getID(object.properties)); } glm::vec3 translation; glm::vec3 rotationOffset; @@ -1099,11 +1099,11 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) geometry.leftHandJointIndex = modelIDs.indexOf(jointLeftHandID); geometry.rightHandJointIndex = modelIDs.indexOf(jointRightHandID); - foreach (const QString& id, jointLeftFingertipIDs) { - geometry.leftFingertipJointIndices.append(modelIDs.indexOf(id)); + foreach (const QString& id, jointLeftFingerIDs) { + geometry.leftFingerJointIndices.append(modelIDs.indexOf(id)); } - foreach (const QString& id, jointRightFingertipIDs) { - geometry.rightFingertipJointIndices.append(modelIDs.indexOf(id)); + foreach (const QString& id, jointRightFingerIDs) { + geometry.rightFingerJointIndices.append(modelIDs.indexOf(id)); } // extract the translation component of the neck transform diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 962986b7f1..2b5134c951 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -137,8 +137,8 @@ public: int leftHandJointIndex; int rightHandJointIndex; - QVector leftFingertipJointIndices; - QVector rightFingertipJointIndices; + QVector leftFingerJointIndices; + QVector rightFingerJointIndices; glm::vec3 neckPivot; From a29dfe9c6346e3eff81a8f082603e4f0023cc2ef Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 16:14:23 -0800 Subject: [PATCH 15/22] Yet another finger attempt. --- interface/src/avatar/SkeletonModel.cpp | 33 ++++++++++--------- interface/src/avatar/SkeletonModel.h | 3 +- interface/src/renderer/FBXReader.cpp | 45 ++++++++++++++++++-------- interface/src/renderer/FBXReader.h | 3 ++ 4 files changed, 54 insertions(+), 30 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 660ef6ee5a..c6ea16f9ad 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -62,12 +62,15 @@ void SkeletonModel::simulate(float deltaTime) { } else if (leftPalmIndex == rightPalmIndex) { // right hand only - applyPalmData(geometry.rightHandJointIndex, geometry.rightFingerJointIndices, hand.getPalms()[leftPalmIndex]); + applyPalmData(geometry.rightHandJointIndex, geometry.rightFingerJointIndices, geometry.rightFingertipJointIndices, + hand.getPalms()[leftPalmIndex]); restoreLeftHandPosition(HAND_RESTORATION_RATE); } else { - applyPalmData(geometry.leftHandJointIndex, geometry.leftFingerJointIndices, hand.getPalms()[leftPalmIndex]); - applyPalmData(geometry.rightHandJointIndex, geometry.rightFingerJointIndices, hand.getPalms()[rightPalmIndex]); + applyPalmData(geometry.leftHandJointIndex, geometry.leftFingerJointIndices, geometry.leftFingertipJointIndices, + hand.getPalms()[leftPalmIndex]); + applyPalmData(geometry.rightHandJointIndex, geometry.rightFingerJointIndices, geometry.rightFingertipJointIndices, + hand.getPalms()[rightPalmIndex]); } } @@ -131,7 +134,8 @@ bool operator<(const IndexValue& firstIndex, const IndexValue& secondIndex) { return firstIndex.value < secondIndex.value; } -void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJointIndices, PalmData& palm) { +void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJointIndices, + const QVector& fingertipJointIndices, PalmData& palm) { const FBXGeometry& geometry = _geometry->getFBXGeometry(); setJointPosition(jointIndex, palm.getPosition()); float sign = (jointIndex == geometry.rightHandJointIndex) ? 1.0f : -1.0f; @@ -152,22 +156,19 @@ void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJoin } qSort(fingerIndices.begin(), fingerIndices.end()); - // likewise with the joint indices and relative z - QVector jointIndices; - foreach (int index, fingerJointIndices) { - glm::vec3 position = glm::inverse(_rotation) * extractTranslation(geometry.joints.at(index).bindTransform); - IndexValue indexValue = { index, position.z * -sign }; - jointIndices.append(indexValue); - } - qSort(jointIndices.begin(), jointIndices.end()); - // match them up as best we can - float proportion = fingerIndices.size() / (float)jointIndices.size(); - for (int i = 0; i < jointIndices.size(); i++) { + float proportion = fingerIndices.size() / (float)fingerJointIndices.size(); + for (int i = 0; i < fingerJointIndices.size(); i++) { int fingerIndex = fingerIndices.at(roundf(i * proportion)).index; glm::vec3 fingerVector = palm.getFingers()[fingerIndex].getTipPosition() - palm.getFingers()[fingerIndex].getRootPosition(); - setJointRotation(jointIndices.at(i).index, rotationBetween(palmRotation * glm::vec3(-sign, 0.0f, 0.0f), fingerVector) * palmRotation, true); + + int fingerJointIndex = fingerJointIndices.at(i); + int fingertipJointIndex = fingertipJointIndices.at(i); + glm::vec3 jointVector = extractTranslation(geometry.joints.at(fingertipJointIndex).bindTransform) - + extractTranslation(geometry.joints.at(fingerJointIndex).bindTransform); + + setJointRotation(fingerJointIndex, rotationBetween(palmRotation * jointVector, fingerVector) * palmRotation, true); } } diff --git a/interface/src/avatar/SkeletonModel.h b/interface/src/avatar/SkeletonModel.h index 6d6b2d148e..1d7a7cc9c0 100644 --- a/interface/src/avatar/SkeletonModel.h +++ b/interface/src/avatar/SkeletonModel.h @@ -28,7 +28,8 @@ public: protected: - void applyPalmData(int jointIndex, const QVector& fingerJointIndices, PalmData& palm); + void applyPalmData(int jointIndex, const QVector& fingerJointIndices, + const QVector& fingertipJointIndices, PalmData& palm); /// Updates the state of the joint at the specified index. virtual void updateJointState(int index); diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 07b2b1a04e..f0ed4f87b1 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -726,6 +726,17 @@ void setTangents(FBXMesh& mesh, int firstIndex, int secondIndex) { -glm::degrees(atan2f(-texCoordDelta.t, texCoordDelta.s)), normal) * glm::normalize(bitangent), normal); } +QVector getIndices(const QVector ids, QVector modelIDs) { + QVector indices; + foreach (const QString& id, ids) { + int index = modelIDs.indexOf(id); + if (index != -1) { + indices.append(index); + } + } + return indices; +} + FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) { QHash meshes; QVector blendshapes; @@ -749,6 +760,8 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QString jointRightHandName = processID(joints.value("jointRightHand", "jointRightHand").toString()); QVariantList jointLeftFingerNames = joints.values("jointLeftFinger"); QVariantList jointRightFingerNames = joints.values("jointRightFinger"); + QVariantList jointLeftFingertipNames = joints.values("jointLeftFingertip"); + QVariantList jointRightFingertipNames = joints.values("jointRightFingertip"); QString jointEyeLeftID; QString jointEyeRightID; QString jointNeckID; @@ -757,8 +770,10 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) QString jointHeadID; QString jointLeftHandID; QString jointRightHandID; - QStringList jointLeftFingerIDs; - QStringList jointRightFingerIDs; + QVector jointLeftFingerIDs(jointLeftFingerNames.size()); + QVector jointRightFingerIDs(jointRightFingerNames.size()); + QVector jointLeftFingertipIDs(jointLeftFingertipNames.size()); + QVector jointRightFingertipIDs(jointRightFingertipNames.size()); QVariantHash blendshapeMappings = mapping.value("bs").toHash(); QHash > blendshapeIndices; @@ -815,6 +830,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } else { name = getID(object.properties); } + int index; if (name == jointEyeLeftName || name == "EyeL" || name == "joint_Leye") { jointEyeLeftID = getID(object.properties); @@ -839,11 +855,17 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } else if (name == jointRightHandName) { jointRightHandID = getID(object.properties); - } else if (jointLeftFingerNames.contains(name)) { - jointLeftFingerIDs.append(getID(object.properties)); + } else if ((index = jointLeftFingerNames.indexOf(name)) != -1) { + jointLeftFingerIDs[index] = getID(object.properties); - } else if (jointRightFingerNames.contains(name)) { - jointRightFingerIDs.append(getID(object.properties)); + } else if ((index = jointRightFingerNames.indexOf(name)) != -1) { + jointRightFingerIDs[index] = getID(object.properties); + + } else if ((index = jointLeftFingertipNames.indexOf(name)) != -1) { + jointLeftFingertipIDs[index] = getID(object.properties); + + } else if ((index = jointRightFingertipNames.indexOf(name)) != -1) { + jointRightFingertipIDs[index] = getID(object.properties); } glm::vec3 translation; glm::vec3 rotationOffset; @@ -1098,13 +1120,10 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) geometry.headJointIndex = modelIDs.indexOf(jointHeadID); geometry.leftHandJointIndex = modelIDs.indexOf(jointLeftHandID); geometry.rightHandJointIndex = modelIDs.indexOf(jointRightHandID); - - foreach (const QString& id, jointLeftFingerIDs) { - geometry.leftFingerJointIndices.append(modelIDs.indexOf(id)); - } - foreach (const QString& id, jointRightFingerIDs) { - geometry.rightFingerJointIndices.append(modelIDs.indexOf(id)); - } + geometry.leftFingerJointIndices = getIndices(jointLeftFingerIDs, modelIDs); + geometry.rightFingerJointIndices = getIndices(jointRightFingerIDs, modelIDs); + geometry.leftFingertipJointIndices = getIndices(jointLeftFingertipIDs, modelIDs); + geometry.rightFingertipJointIndices = getIndices(jointRightFingertipIDs, modelIDs); // extract the translation component of the neck transform if (geometry.neckJointIndex != -1) { diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 2b5134c951..48366c09b9 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -140,6 +140,9 @@ public: QVector leftFingerJointIndices; QVector rightFingerJointIndices; + QVector leftFingertipJointIndices; + QVector rightFingertipJointIndices; + glm::vec3 neckPivot; QVector attachments; From d35356348b7072210aa829c945a262fac07375ad Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 16:24:46 -0800 Subject: [PATCH 16/22] Demagicked a couple of numbers. --- interface/src/avatar/Avatar.cpp | 3 ++- interface/src/avatar/SkeletonModel.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 9f1eb263e7..929b5b600b 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -665,7 +665,8 @@ void Avatar::updateArmIKAndConstraints(float deltaTime, AvatarJointID fingerTipJ float distance = glm::length(armVector); // don't let right hand get dragged beyond maximum arm length... - float armLength = _maxArmLength * 0.75f; + const float ARM_RETRACTION = 0.75f; + float armLength = _maxArmLength * ARM_RETRACTION; if (distance > armLength) { // reset right hand to be constrained to maximum arm length fingerJoint.position = shoulderJoint.position; diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index c6ea16f9ad..7da4cf14b8 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -140,7 +140,7 @@ void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJoin setJointPosition(jointIndex, palm.getPosition()); float sign = (jointIndex == geometry.rightHandJointIndex) ? 1.0f : -1.0f; glm::quat palmRotation = rotationBetween(_rotation * IDENTITY_UP, -palm.getNormal()) * _rotation * - glm::angleAxis(90.0f, 0.0f, sign, 0.0f); + glm::angleAxis(90.0f, 0.0f, sign, 0.0f); // ninety degree rotation to face fingers forward from bind pose setJointRotation(jointIndex, palmRotation, true); // no point in continuing if there are no fingers From 55d31ee7f332367ab42c8e84d7354dbce90567b7 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 16:37:51 -0800 Subject: [PATCH 17/22] Render the body in first-person mode so that we can see our hands. --- interface/src/avatar/MyAvatar.cpp | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 0a95e78b91..81636d4701 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -603,13 +603,6 @@ float MyAvatar::getBallRenderAlpha(int ball, bool lookingInMirror) const { void MyAvatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { - if (Application::getInstance()->getCamera()->getMode() == CAMERA_MODE_FIRST_PERSON && !lookingInMirror) { - // Dont display body, only the hand - _hand.render(lookingInMirror); - - return; - } - if (_head.getVideoFace().isFullFrame()) { // Render the full-frame video float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); @@ -684,11 +677,11 @@ void MyAvatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { } } else { // Render the body's voxels and head + if (!_skeletonModel.render(1.0f)) { + _voxels.render(false); + } float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); if (alpha > 0.0f) { - if (!_skeletonModel.render(alpha)) { - _voxels.render(false); - } _head.render(alpha, true); } } From 4bf4accba7c0560ca8694a1bd9a5df839a44bf8f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 16:57:07 -0800 Subject: [PATCH 18/22] Rotate the palm according to the average finger direction. --- interface/src/avatar/SkeletonModel.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 7da4cf14b8..e7772e5cce 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -148,14 +148,26 @@ void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJoin return; } - // sort the finger indices by raw x + // sort the finger indices by raw x, get the average direction QVector fingerIndices; + glm::vec3 direction; for (int i = 0; i < palm.getNumFingers(); i++) { + glm::vec3 fingerVector = palm.getFingers()[i].getTipPosition() - palm.getFingers()[i].getRootPosition(); + float length = glm::length(fingerVector); + if (length > EPSILON) { + direction += fingerVector / length; + } IndexValue indexValue = { i, palm.getFingers()[i].getTipRawPosition().x }; fingerIndices.append(indexValue); } qSort(fingerIndices.begin(), fingerIndices.end()); + // rotate palm according to average finger direction + float directionLength = glm::length(direction); + if (directionLength > EPSILON) { + palmRotation = rotationBetween(palmRotation * glm::vec3(sign, 0.0f, 0.0f), direction) * palmRotation; + } + // match them up as best we can float proportion = fingerIndices.size() / (float)fingerJointIndices.size(); for (int i = 0; i < fingerJointIndices.size(); i++) { From f5889934a20f39a89336e154cfa5b9a08df634c3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 17:01:29 -0800 Subject: [PATCH 19/22] Derp, have to set the rotation after adjusting it. --- interface/src/avatar/SkeletonModel.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index e7772e5cce..9e0548fe17 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -141,13 +141,7 @@ void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJoin float sign = (jointIndex == geometry.rightHandJointIndex) ? 1.0f : -1.0f; glm::quat palmRotation = rotationBetween(_rotation * IDENTITY_UP, -palm.getNormal()) * _rotation * glm::angleAxis(90.0f, 0.0f, sign, 0.0f); // ninety degree rotation to face fingers forward from bind pose - setJointRotation(jointIndex, palmRotation, true); - // no point in continuing if there are no fingers - if (palm.getNumFingers() == 0 || fingerJointIndices.isEmpty()) { - return; - } - // sort the finger indices by raw x, get the average direction QVector fingerIndices; glm::vec3 direction; @@ -165,9 +159,15 @@ void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJoin // rotate palm according to average finger direction float directionLength = glm::length(direction); if (directionLength > EPSILON) { - palmRotation = rotationBetween(palmRotation * glm::vec3(sign, 0.0f, 0.0f), direction) * palmRotation; + palmRotation = rotationBetween(palmRotation * glm::vec3(-sign, 0.0f, 0.0f), direction) * palmRotation; } + setJointRotation(jointIndex, palmRotation, true); + // no point in continuing if there are no fingers + if (palm.getNumFingers() == 0 || fingerJointIndices.isEmpty()) { + return; + } + // match them up as best we can float proportion = fingerIndices.size() / (float)fingerJointIndices.size(); for (int i = 0; i < fingerJointIndices.size(); i++) { From ebe3ef4b22f54cbe6915551a0574af12e93601c9 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 17:20:52 -0800 Subject: [PATCH 20/22] Toggle for the leap hands. --- interface/src/Menu.cpp | 1 + interface/src/Menu.h | 1 + interface/src/avatar/Hand.cpp | 5 ++--- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 3d1b7b7bf1..48f5a4ed3c 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -355,6 +355,7 @@ Menu::Menu() : QMenu* raveGloveOptionsMenu = developerMenu->addMenu("Rave Glove Options"); addCheckableActionToQMenuAndActionHash(raveGloveOptionsMenu, MenuOption::SimulateLeapHand); + addCheckableActionToQMenuAndActionHash(raveGloveOptionsMenu, MenuOption::DisplayLeapHands, 0, true); addCheckableActionToQMenuAndActionHash(raveGloveOptionsMenu, MenuOption::TestRaveGlove); QMenu* trackingOptionsMenu = developerMenu->addMenu("Tracking Options"); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 30ef055447..dc8f6051a4 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -160,6 +160,7 @@ namespace MenuOption { const QString DisableConstantCulling = "Disable Constant Culling"; const QString DisableFastVoxelPipeline = "Disable Fast Voxel Pipeline"; const QString DisplayFrustum = "Display Frustum"; + const QString DisplayLeapHands = "Display Leap Hands"; const QString DontRenderVoxels = "Don't call _voxels.render()"; const QString DontCallOpenGLForVoxels = "Don't call glDrawRangeElementsEXT() for Voxels"; const QString EchoAudio = "Echo Audio"; diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index dda948d92f..a9397913bb 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -11,11 +11,10 @@ #include "Application.h" #include "Avatar.h" #include "Hand.h" +#include "Menu.h" #include "Util.h" #include "renderer/ProgramObject.h" -const bool SHOW_LEAP_HAND = true; - using namespace std; Hand::Hand(Avatar* owningAvatar) : @@ -139,7 +138,7 @@ void Hand::render(bool lookingInMirror) { calculateGeometry(); - if ( SHOW_LEAP_HAND ) { + if (Menu::getInstance()->isOptionChecked(MenuOption::DisplayLeapHands)) { if (!isRaveGloveActive()) { renderLeapFingerTrails(); } From 11fedb23cde5dee67d684702d1692d3b13281190 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 17:51:54 -0800 Subject: [PATCH 21/22] Try finding the fingers using the rotation about the palm. --- interface/src/avatar/SkeletonModel.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 9e0548fe17..92b9f7aada 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -146,12 +146,13 @@ void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJoin QVector fingerIndices; glm::vec3 direction; for (int i = 0; i < palm.getNumFingers(); i++) { - glm::vec3 fingerVector = palm.getFingers()[i].getTipPosition() - palm.getFingers()[i].getRootPosition(); + glm::vec3 fingerVector = palm.getFingers()[i].getTipPosition() - palm.getPosition(); float length = glm::length(fingerVector); if (length > EPSILON) { direction += fingerVector / length; } - IndexValue indexValue = { i, palm.getFingers()[i].getTipRawPosition().x }; + fingerVector = glm::inverse(palmRotation) * fingerVector; + IndexValue indexValue = { i, atan2f(fingerVector.z, fingerVector.x) }; fingerIndices.append(indexValue); } qSort(fingerIndices.begin(), fingerIndices.end()); From 62134ebc4ca3bbbcc383a4260037e9190e4ba0df Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 7 Nov 2013 18:00:21 -0800 Subject: [PATCH 22/22] Sign fix. --- interface/src/avatar/SkeletonModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 92b9f7aada..adf708a9bd 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -151,7 +151,7 @@ void SkeletonModel::applyPalmData(int jointIndex, const QVector& fingerJoin if (length > EPSILON) { direction += fingerVector / length; } - fingerVector = glm::inverse(palmRotation) * fingerVector; + fingerVector = glm::inverse(palmRotation) * fingerVector * -sign; IndexValue indexValue = { i, atan2f(fingerVector.z, fingerVector.x) }; fingerIndices.append(indexValue); }