Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-11 11:33:07 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #10040 from ZappoMan/addKinectHandRotation

Browse source 
add support for hand rotations to the kinect plugin
This commit is contained in:
Andrew Meadows 2017-03-29 10:25:04 -07:00 committed by GitHub
commit 985924a476
2 changed files with 104 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

124
plugins/hifiKinect/src/KinectPlugin.cpp
View file

@ -239,6 +239,14 @@ void KinectPlugin::init() {
auto preference = new CheckPreference(KINECT_PLUGIN, "Enabled", getter, setter); auto preference = new CheckPreference(KINECT_PLUGIN, "Enabled", getter, setter);
preferences->addPreference(preference); preferences->addPreference(preference);
} }
{
auto debugGetter = [this]()->bool { return _enabled; };
auto debugSetter = [this](bool value) {
_debug = value; saveSettings();
};
auto preference = new CheckPreference(KINECT_PLUGIN, "Extra Debugging", debugGetter, debugSetter);
preferences->addPreference(preference);
}
} }
bool KinectPlugin::isSupported() const { bool KinectPlugin::isSupported() const {
@ -389,37 +397,110 @@ void KinectPlugin::ProcessBody(INT64 time, int bodyCount, IBody** bodies) {
if (SUCCEEDED(hr)) { if (SUCCEEDED(hr)) {
auto jointCount = _countof(joints); auto jointCount = _countof(joints);
//qDebug() << __FUNCTION__ << "nBodyCount:" << nBodyCount << "body:" << i << "jointCount:" << jointCount; if (_debug) {
qDebug() << __FUNCTION__ << "nBodyCount:" << bodyCount << "body:" << i << "jointCount:" << jointCount;
}
for (int j = 0; j < jointCount; ++j) { for (int j = 0; j < jointCount; ++j) {
//QString jointName = kinectJointNames[joints[j].JointType];
glm::vec3 jointPosition { joints[j].Position.X, glm::vec3 jointPosition { joints[j].Position.X,
joints[j].Position.Y, joints[j].Position.Y,
joints[j].Position.Z }; joints[j].Position.Z };
// Kinect Documentation is unclear on what these orientations are, are they absolute? // This is the rotation in the kinect camera/sensor frame... we adjust that in update...
// or are the relative to the parent bones. It appears as if it has changed between the // NOTE: glm::quat(W!!!, x, y, z)... not (x,y,z,w)!!!
// older 1.x SDK and the 2.0 sdk glm::quat jointOrientation { jointOrientations[j].Orientation.w,
// jointOrientations[j].Orientation.x,
// https://social.msdn.microsoft.com/Forums/en-US/31c9aff6-7dab-433d-9af9-59942dfd3d69/kinect-v20-preview-sdk-jointorientation-vs-boneorientation?forum=kinectv2sdk
// seems to suggest these are absolute...
// "These quaternions are absolute, so you can take a mesh in local space, transform it by the quaternion,
// and it will match the exact orientation of the bone. If you want relative orientation quaternion, you
// can multiply the absolute quaternion by the inverse of the parent joint's quaternion."
//
// - Bone direction(Y green) - always matches the skeleton.
// - Normal(Z blue) - joint roll, perpendicular to the bone
// - Binormal(X orange) - perpendicular to the bone and normal
glm::quat jointOrientation { jointOrientations[j].Orientation.x,
jointOrientations[j].Orientation.y, jointOrientations[j].Orientation.y,
jointOrientations[j].Orientation.z, jointOrientations[j].Orientation.z };
jointOrientations[j].Orientation.w };
if (_debug) {
QString jointName = kinectJointNames[joints[j].JointType];
qDebug() << __FUNCTION__ << "joint[" << j << "]:" << jointName
<< "position:" << jointPosition
<< "orientation:" << jointOrientation
<< "isTracked:" << (joints[j].TrackingState != TrackingState_NotTracked);
}
// filling in the _joints data... // filling in the _joints data...
if (joints[j].TrackingState != TrackingState_NotTracked) { if (joints[j].TrackingState != TrackingState_NotTracked) {
_joints[j].position = jointPosition; _joints[j].position = jointPosition;
// Kinect Documentation...
//
// https://social.msdn.microsoft.com/Forums/en-US/31c9aff6-7dab-433d-9af9-59942dfd3d69/kinect-v20-preview-sdk-jointorientation-vs-boneorientation?forum=kinectv2sdk
// seems to suggest these are absolute...
// "These quaternions are absolute, so you can take a mesh in local space, transform it by the quaternion,
// and it will match the exact orientation of the bone. If you want relative orientation quaternion, you
// can multiply the absolute quaternion by the inverse of the parent joint's quaternion."
//
// This is consistent with our findings, but does not include "enough information"
// - Bone direction(Y green) - always matches the skeleton.
// - Normal(Z blue) - joint roll, perpendicular to the bone
// - Binormal(X orange) - perpendicular to the bone and normal
// NOTE: Common notation of vectors on paper...
// (+) is the back of the arrow - this vector is pointing into the page
// (o) is the point of the arrow - this vector is pointing out of the page
//
// From ABOVE the kinect coordinate frame looks like this:
//
// Assuming standing facing the kinect camera
// Right Hand with fingers pointing up (green/y)
// thumb pointing behind body (blue/z)
// palm facing the head (point out back of my hand, red/x)
//
// The identity rotation relative to the cameras frame... (the joint data from SDK)
//
// y | | | |
// | | | | |
// | | |
// z----(o) \ |right|
// x \_ |
// | |
// | |
//
// Expected... identity rotation for left hand..... [to be verified]
// Left Hand with fingers pointing up (green/y)
// thumb pointing forward (blue/z)
// palm facing outward away from head (point out back of my hand, red/x)
//
// Our desired coordinate system...
// "the local coordinate of the palm in our system"...
//
// From ABOVE the hand canonical axes look like this:
//
//
// | | | | y | | | |
// | | | | | | | | |
// | | | | |
// |left | / x----(+) \ |right|
// | _/ z \_ |
// | | | |
// | | | |
//
// Right hand rule... make the hitch hiking sign...
// thumb points in direction of the axis you want to rotate around
// fisted fingers curl in positive rotation direction....
//
// To transform from Kinect to our RIGHT Hand.... Negative 90 deg around Y
//
// FIXME -- Double check if JointType_HandRight vs JointType_WristRight is actually
// the joint we want to be using!!
//
//_joints[j].orientation = jointOrientation; //_joints[j].orientation = jointOrientation;
if (joints[j].JointType == JointType_HandRight) {
static const quat kinectToHandRight = glm::angleAxis(-PI / 2.0f, Vectors::UNIT_Y);
_joints[j].orientation = jointOrientation * kinectToHandRight;
} else if (joints[j].JointType == JointType_HandLeft) {
// To transform from Kinect to our LEFT Hand.... Postive 90 deg around Y
static const quat kinectToHandLeft = glm::angleAxis(PI / 2.0f, Vectors::UNIT_Y);
_joints[j].orientation = jointOrientation * kinectToHandLeft;
} else {
_joints[j].orientation = jointOrientation;
}
} }
} }
} }
@ -482,6 +563,7 @@ void KinectPlugin::saveSettings() const {
settings.beginGroup(idString); settings.beginGroup(idString);
{ {
settings.setValue(QString("enabled"), _enabled); settings.setValue(QString("enabled"), _enabled);
settings.setValue(QString("extraDebug"), _debug);
} }
settings.endGroup(); settings.endGroup();
} }
@ -541,8 +623,8 @@ void KinectPlugin::InputDevice::update(float deltaTime, const controller::InputC
continue; continue;
} }
// FIXME - determine the correct orientation transform // Note: we want our rotations presenting in the AVATAR frame, so we need to adjust that here.
glm::quat rot = joints[i].orientation; glm::quat rot = controllerToAvatarRotation * joints[i].orientation;
if (i < prevJoints.size()) { if (i < prevJoints.size()) {
linearVel = (pos - (prevJoints[i].position * METERS_PER_CENTIMETER)) / deltaTime; // m/s linearVel = (pos - (prevJoints[i].position * METERS_PER_CENTIMETER)) / deltaTime; // m/s

1
plugins/hifiKinect/src/KinectPlugin.h
View file

@ -89,6 +89,7 @@ protected:
static const char* KINECT_ID_STRING; static const char* KINECT_ID_STRING;
bool _enabled { false }; bool _enabled { false };
bool _debug { false };
mutable bool _initialized { false }; mutable bool _initialized { false };
// copy of data directly from the KinectDataReader SDK // copy of data directly from the KinectDataReader SDK