Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-07-26 07:26:37 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #764 from Atlante45/fix_key_press_event

Browse source 
Fix key press event
This commit is contained in:
Stephen Birarda 2013-08-02 18:06:39 -07:00
commit dbc1152fba
3 changed files with 129 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
interface/src/Application.cpp
View file

@ -571,9 +571,8 @@ void Application::keyPressEvent(QKeyEvent* event) {
_myAvatar.getHand().setRaveGloveEffectsMode((QKeyEvent*)event); _myAvatar.getHand().setRaveGloveEffectsMode((QKeyEvent*)event);
} }
bool alt = event->modifiers().testFlag(Qt::AltModifier); bool isMeta = event->modifiers().testFlag(Qt::MetaModifier);
bool meta = event->modifiers().testFlag(Qt::MetaModifier); bool isShifted = event->modifiers().testFlag(Qt::ShiftModifier);
bool shifted = event->modifiers().testFlag(Qt::ShiftModifier);
switch (event->key()) { switch (event->key()) {
case Qt::Key_BracketLeft: case Qt::Key_BracketLeft:
_viewFrustumOffsetYaw -= 0.5; _viewFrustumOffsetYaw -= 0.5;
@ -646,9 +645,9 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_C: case Qt::Key_C:
if (shifted) { if (isShifted) {
_occlusionCulling->trigger(); _occlusionCulling->trigger();
} else if (meta) { } else if (isMeta) {
chooseVoxelPaintColor(); chooseVoxelPaintColor();
} else { } else {
_myAvatar.setDriveKeys(DOWN, 1); _myAvatar.setDriveKeys(DOWN, 1);
@ -660,7 +659,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_S: case Qt::Key_S:
if (shifted) { if (isShifted) {
doTreeStats(); doTreeStats();
} else { } else {
_myAvatar.setDriveKeys(BACK, 1); _myAvatar.setDriveKeys(BACK, 1);
@ -673,7 +672,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_G: case Qt::Key_G:
if (shifted) { if (isShifted) {
_gravityUse->trigger(); _gravityUse->trigger();
} else { } else {
_eyedropperMode->trigger(); _eyedropperMode->trigger();
@ -681,7 +680,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_A: case Qt::Key_A:
if (shifted) { if (isShifted) {
_renderAtmosphereOn->trigger(); _renderAtmosphereOn->trigger();
} else { } else {
_myAvatar.setDriveKeys(ROT_LEFT, 1); _myAvatar.setDriveKeys(ROT_LEFT, 1);
@ -701,23 +700,23 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_Up: case Qt::Key_Up:
_myAvatar.setDriveKeys(shifted ? UP : FWD, 1); _myAvatar.setDriveKeys(isShifted ? UP : FWD, 1);
break; break;
case Qt::Key_Down: case Qt::Key_Down:
_myAvatar.setDriveKeys(shifted ? DOWN : BACK, 1); _myAvatar.setDriveKeys(isShifted ? DOWN : BACK, 1);
break; break;
case Qt::Key_Left: case Qt::Key_Left:
_myAvatar.setDriveKeys(shifted ? LEFT : ROT_LEFT, 1); _myAvatar.setDriveKeys(isShifted ? LEFT : ROT_LEFT, 1);
break; break;
case Qt::Key_Right: case Qt::Key_Right:
_myAvatar.setDriveKeys(shifted ? RIGHT : ROT_RIGHT, 1); _myAvatar.setDriveKeys(isShifted ? RIGHT : ROT_RIGHT, 1);
break; break;
case Qt::Key_I: case Qt::Key_I:
if (shifted) { if (isShifted) {
_myCamera.setEyeOffsetOrientation(glm::normalize( _myCamera.setEyeOffsetOrientation(glm::normalize(
glm::quat(glm::vec3(0.002f, 0, 0)) * _myCamera.getEyeOffsetOrientation())); glm::quat(glm::vec3(0.002f, 0, 0)) * _myCamera.getEyeOffsetOrientation()));
} else { } else {
@ -727,7 +726,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_K: case Qt::Key_K:
if (shifted) { if (isShifted) {
_myCamera.setEyeOffsetOrientation(glm::normalize( _myCamera.setEyeOffsetOrientation(glm::normalize(
glm::quat(glm::vec3(-0.002f, 0, 0)) * _myCamera.getEyeOffsetOrientation())); glm::quat(glm::vec3(-0.002f, 0, 0)) * _myCamera.getEyeOffsetOrientation()));
} else { } else {
@ -737,7 +736,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_J: case Qt::Key_J:
if (shifted) { if (isShifted) {
_myCamera.setEyeOffsetOrientation(glm::normalize( _myCamera.setEyeOffsetOrientation(glm::normalize(
glm::quat(glm::vec3(0, 0.002f, 0)) * _myCamera.getEyeOffsetOrientation())); glm::quat(glm::vec3(0, 0.002f, 0)) * _myCamera.getEyeOffsetOrientation()));
} else { } else {
@ -747,7 +746,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_M: case Qt::Key_M:
if (shifted) { if (isShifted) {
_myCamera.setEyeOffsetOrientation(glm::normalize( _myCamera.setEyeOffsetOrientation(glm::normalize(
glm::quat(glm::vec3(0, -0.002f, 0)) * _myCamera.getEyeOffsetOrientation())); glm::quat(glm::vec3(0, -0.002f, 0)) * _myCamera.getEyeOffsetOrientation()));
} else { } else {
@ -757,7 +756,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_U: case Qt::Key_U:
if (shifted) { if (isShifted) {
_myCamera.setEyeOffsetOrientation(glm::normalize( _myCamera.setEyeOffsetOrientation(glm::normalize(
glm::quat(glm::vec3(0, 0, -0.002f)) * _myCamera.getEyeOffsetOrientation())); glm::quat(glm::vec3(0, 0, -0.002f)) * _myCamera.getEyeOffsetOrientation()));
} else { } else {
@ -767,7 +766,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
break; break;
case Qt::Key_Y: case Qt::Key_Y:
if (shifted) { if (isShifted) {
_myCamera.setEyeOffsetOrientation(glm::normalize( _myCamera.setEyeOffsetOrientation(glm::normalize(
glm::quat(glm::vec3(0, 0, 0.002f)) * _myCamera.getEyeOffsetOrientation())); glm::quat(glm::vec3(0, 0, 0.002f)) * _myCamera.getEyeOffsetOrientation()));
} else { } else {
@ -782,14 +781,14 @@ void Application::keyPressEvent(QKeyEvent* event) {
_lookingInMirror->trigger(); _lookingInMirror->trigger();
break; break;
case Qt::Key_F: case Qt::Key_F:
if (shifted) { if (isShifted) {
_frustumOn->trigger(); _frustumOn->trigger();
} else { } else {
_fullScreenMode->trigger(); _fullScreenMode->trigger();
} }
break; break;
case Qt::Key_V: case Qt::Key_V:
if (shifted) { if (isShifted) {
_renderVoxels->trigger(); _renderVoxels->trigger();
} else { } else {
_addVoxelMode->trigger(); _addVoxelMode->trigger();
@ -799,7 +798,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
_manualFirstPerson->trigger(); _manualFirstPerson->trigger();
break; break;
case Qt::Key_R: case Qt::Key_R:
if (shifted) { if (isShifted) {
_frustumRenderModeAction->trigger(); _frustumRenderModeAction->trigger();
} else { } else {
_deleteVoxelMode->trigger(); _deleteVoxelMode->trigger();
@ -809,7 +808,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
_colorVoxelMode->trigger(); _colorVoxelMode->trigger();
break; break;
case Qt::Key_O: case Qt::Key_O:
if (shifted) { if (isShifted) {
_viewFrustumFromOffset->trigger(); _viewFrustumFromOffset->trigger();
} else { } else {
_selectVoxelMode->trigger(); _selectVoxelMode->trigger();
@ -825,14 +824,10 @@ void Application::keyPressEvent(QKeyEvent* event) {
} }
break; break;
case Qt::Key_Plus: case Qt::Key_Plus:
if (alt) { increaseAvatarSize();
increaseAvatarSize();
}
break; break;
case Qt::Key_Minus: case Qt::Key_Minus:
if (alt) { decreaseAvatarSize();
decreaseAvatarSize();
}
break; break;
case Qt::Key_1: case Qt::Key_1:
@ -1930,8 +1925,8 @@ void Application::initMenu() {
"First Person", this, SLOT(setRenderFirstPerson(bool)), Qt::Key_P))->setCheckable(true); "First Person", this, SLOT(setRenderFirstPerson(bool)), Qt::Key_P))->setCheckable(true);
(_manualThirdPerson = renderMenu->addAction( (_manualThirdPerson = renderMenu->addAction(
"Third Person", this, SLOT(setRenderThirdPerson(bool))))->setCheckable(true); "Third Person", this, SLOT(setRenderThirdPerson(bool))))->setCheckable(true);
renderMenu->addAction("Increase Avatar Size", this, SLOT(increaseAvatarSize()), Qt::ALT | Qt::Key_Plus); renderMenu->addAction("Increase Avatar Size", this, SLOT(increaseAvatarSize()), Qt::Key_Plus);
renderMenu->addAction("Decrease Avatar Size", this, SLOT(decreaseAvatarSize()), Qt::ALT | Qt::Key_Minus); renderMenu->addAction("Decrease Avatar Size", this, SLOT(decreaseAvatarSize()), Qt::Key_Minus);
QMenu* toolsMenu = menuBar->addMenu("Tools"); QMenu* toolsMenu = menuBar->addMenu("Tools");

160
interface/src/Webcam.cpp
View file

@ -47,11 +47,11 @@ void Webcam::setEnabled(bool enabled) {
_grabberThread.start(); _grabberThread.start();
_startTimestamp = 0; _startTimestamp = 0;
_frameCount = 0; _frameCount = 0;
// let the grabber know we're ready for the first frame // let the grabber know we're ready for the first frame
QMetaObject::invokeMethod(_grabber, "reset"); QMetaObject::invokeMethod(_grabber, "reset");
QMetaObject::invokeMethod(_grabber, "grabFrame"); QMetaObject::invokeMethod(_grabber, "grabFrame");
} else { } else {
QMetaObject::invokeMethod(_grabber, "shutdown"); QMetaObject::invokeMethod(_grabber, "shutdown");
_active = false; _active = false;
@ -63,7 +63,7 @@ const float UNINITIALIZED_FACE_DEPTH = 0.0f;
void Webcam::reset() { void Webcam::reset() {
_initialFaceRect = RotatedRect(); _initialFaceRect = RotatedRect();
_initialFaceDepth = UNINITIALIZED_FACE_DEPTH; _initialFaceDepth = UNINITIALIZED_FACE_DEPTH;
if (_enabled) { if (_enabled) {
// send a message to the grabber // send a message to the grabber
QMetaObject::invokeMethod(_grabber, "reset"); QMetaObject::invokeMethod(_grabber, "reset");
@ -80,7 +80,7 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) {
int previewWidth = _textureSize.width * PREVIEW_HEIGHT / _textureSize.height; int previewWidth = _textureSize.width * PREVIEW_HEIGHT / _textureSize.height;
int top = screenHeight - 600; int top = screenHeight - 600;
int left = screenWidth - previewWidth - 10; int left = screenWidth - previewWidth - 10;
glTexCoord2f(0, 0); glTexCoord2f(0, 0);
glVertex2f(left, top); glVertex2f(left, top);
glTexCoord2f(1, 0); glTexCoord2f(1, 0);
@ -90,7 +90,7 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) {
glTexCoord2f(0, 1); glTexCoord2f(0, 1);
glVertex2f(left, top + PREVIEW_HEIGHT); glVertex2f(left, top + PREVIEW_HEIGHT);
glEnd(); glEnd();
if (_depthTextureID != 0) { if (_depthTextureID != 0) {
glBindTexture(GL_TEXTURE_2D, _depthTextureID); glBindTexture(GL_TEXTURE_2D, _depthTextureID);
glBegin(GL_QUADS); glBegin(GL_QUADS);
@ -103,10 +103,10 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) {
glTexCoord2f(0, 1); glTexCoord2f(0, 1);
glVertex2f(left, top); glVertex2f(left, top);
glEnd(); glEnd();
glBindTexture(GL_TEXTURE_2D, 0); glBindTexture(GL_TEXTURE_2D, 0);
glDisable(GL_TEXTURE_2D); glDisable(GL_TEXTURE_2D);
if (!_joints.isEmpty()) { if (!_joints.isEmpty()) {
glColor3f(1.0f, 0.0f, 0.0f); glColor3f(1.0f, 0.0f, 0.0f);
glPointSize(4.0f); glPointSize(4.0f);
@ -125,7 +125,7 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) {
glBindTexture(GL_TEXTURE_2D, 0); glBindTexture(GL_TEXTURE_2D, 0);
glDisable(GL_TEXTURE_2D); glDisable(GL_TEXTURE_2D);
} }
glColor3f(1.0f, 1.0f, 1.0f); glColor3f(1.0f, 1.0f, 1.0f);
glBegin(GL_LINE_LOOP); glBegin(GL_LINE_LOOP);
Point2f facePoints[4]; Point2f facePoints[4];
@ -137,7 +137,7 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) {
glVertex2f(left + facePoints[2].x * xScale, top + facePoints[2].y * yScale); glVertex2f(left + facePoints[2].x * xScale, top + facePoints[2].y * yScale);
glVertex2f(left + facePoints[3].x * xScale, top + facePoints[3].y * yScale); glVertex2f(left + facePoints[3].x * xScale, top + facePoints[3].y * yScale);
glEnd(); glEnd();
const int MAX_FPS_CHARACTERS = 30; const int MAX_FPS_CHARACTERS = 30;
char fps[MAX_FPS_CHARACTERS]; char fps[MAX_FPS_CHARACTERS];
sprintf(fps, "FPS: %d", (int)(roundf(_frameCount * 1000000.0f / (usecTimestampNow() - _startTimestamp)))); sprintf(fps, "FPS: %d", (int)(roundf(_frameCount * 1000000.0f / (usecTimestampNow() - _startTimestamp))));
@ -149,7 +149,7 @@ Webcam::~Webcam() {
// stop the grabber thread // stop the grabber thread
_grabberThread.quit(); _grabberThread.quit();
_grabberThread.wait(); _grabberThread.wait();
delete _grabber; delete _grabber;
} }
@ -166,13 +166,13 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF
0, format, GL_UNSIGNED_BYTE, colorImage.imageData); 0, format, GL_UNSIGNED_BYTE, colorImage.imageData);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
qDebug("Capturing video at %gx%g.\n", _textureSize.width, _textureSize.height); qDebug("Capturing video at %gx%g.\n", _textureSize.width, _textureSize.height);
} else { } else {
glBindTexture(GL_TEXTURE_2D, _colorTextureID); glBindTexture(GL_TEXTURE_2D, _colorTextureID);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _textureSize.width, _textureSize.height, format, glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _textureSize.width, _textureSize.height, format,
GL_UNSIGNED_BYTE, colorImage.imageData); GL_UNSIGNED_BYTE, colorImage.imageData);
} }
if (!depth.empty()) { if (!depth.empty()) {
IplImage depthImage = depth; IplImage depthImage = depth;
glPixelStorei(GL_UNPACK_ROW_LENGTH, depthImage.widthStep); glPixelStorei(GL_UNPACK_ROW_LENGTH, depthImage.widthStep);
@ -183,23 +183,23 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF
GL_LUMINANCE, GL_UNSIGNED_BYTE, depthImage.imageData); GL_LUMINANCE, GL_UNSIGNED_BYTE, depthImage.imageData);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
} else { } else {
glBindTexture(GL_TEXTURE_2D, _depthTextureID); glBindTexture(GL_TEXTURE_2D, _depthTextureID);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _textureSize.width, _textureSize.height, GL_LUMINANCE, glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _textureSize.width, _textureSize.height, GL_LUMINANCE,
GL_UNSIGNED_BYTE, depthImage.imageData); GL_UNSIGNED_BYTE, depthImage.imageData);
} }
} }
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
glBindTexture(GL_TEXTURE_2D, 0); glBindTexture(GL_TEXTURE_2D, 0);
// store our various data, update our frame count for fps computation // store our various data, update our frame count for fps computation
_aspectRatio = aspectRatio; _aspectRatio = aspectRatio;
_faceRect = faceRect; _faceRect = faceRect;
_sending = sending; _sending = sending;
_joints = _skeletonTrackingOn ? joints : JointVector(); _joints = _skeletonTrackingOn ? joints : JointVector();
_frameCount++; _frameCount++;
const int MAX_FPS = 60; const int MAX_FPS = 60;
const int MIN_FRAME_DELAY = 1000000 / MAX_FPS; const int MIN_FRAME_DELAY = 1000000 / MAX_FPS;
uint64_t now = usecTimestampNow(); uint64_t now = usecTimestampNow();
@ -210,14 +210,14 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF
remaining -= (now - _lastFrameTimestamp); remaining -= (now - _lastFrameTimestamp);
} }
_lastFrameTimestamp = now; _lastFrameTimestamp = now;
// see if we have joint data // see if we have joint data
if (!_joints.isEmpty()) { if (!_joints.isEmpty()) {
_estimatedJoints.resize(NUM_AVATAR_JOINTS); _estimatedJoints.resize(NUM_AVATAR_JOINTS);
glm::vec3 origin; glm::vec3 origin;
if (_joints[AVATAR_JOINT_LEFT_HIP].isValid && _joints[AVATAR_JOINT_RIGHT_HIP].isValid) { if (_joints[AVATAR_JOINT_LEFT_HIP].isValid && _joints[AVATAR_JOINT_RIGHT_HIP].isValid) {
origin = glm::mix(_joints[AVATAR_JOINT_LEFT_HIP].position, _joints[AVATAR_JOINT_RIGHT_HIP].position, 0.5f); origin = glm::mix(_joints[AVATAR_JOINT_LEFT_HIP].position, _joints[AVATAR_JOINT_RIGHT_HIP].position, 0.5f);
} else if (_joints[AVATAR_JOINT_TORSO].isValid) { } else if (_joints[AVATAR_JOINT_TORSO].isValid) {
const glm::vec3 TORSO_TO_PELVIS = glm::vec3(0.0f, -0.09f, -0.01f); const glm::vec3 TORSO_TO_PELVIS = glm::vec3(0.0f, -0.09f, -0.01f);
origin = _joints[AVATAR_JOINT_TORSO].position + TORSO_TO_PELVIS; origin = _joints[AVATAR_JOINT_TORSO].position + TORSO_TO_PELVIS;
@ -235,27 +235,27 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF
} }
_estimatedRotation = safeEulerAngles(_estimatedJoints[AVATAR_JOINT_HEAD_BASE].rotation); _estimatedRotation = safeEulerAngles(_estimatedJoints[AVATAR_JOINT_HEAD_BASE].rotation);
_estimatedPosition = _estimatedJoints[AVATAR_JOINT_HEAD_BASE].position; _estimatedPosition = _estimatedJoints[AVATAR_JOINT_HEAD_BASE].position;
} else { } else {
// roll is just the angle of the face rect // roll is just the angle of the face rect
const float ROTATION_SMOOTHING = 0.95f; const float ROTATION_SMOOTHING = 0.95f;
_estimatedRotation.z = glm::mix(_faceRect.angle, _estimatedRotation.z, ROTATION_SMOOTHING); _estimatedRotation.z = glm::mix(_faceRect.angle, _estimatedRotation.z, ROTATION_SMOOTHING);
// determine position based on translation and scaling of the face rect/mean face depth // determine position based on translation and scaling of the face rect/mean face depth
if (_initialFaceRect.size.area() == 0) { if (_initialFaceRect.size.area() == 0) {
_initialFaceRect = _faceRect; _initialFaceRect = _faceRect;
_estimatedPosition = glm::vec3(); _estimatedPosition = glm::vec3();
_initialFaceDepth = midFaceDepth; _initialFaceDepth = midFaceDepth;
} else { } else {
float proportion, z; float proportion, z;
if (midFaceDepth == UNINITIALIZED_FACE_DEPTH) { if (midFaceDepth == UNINITIALIZED_FACE_DEPTH) {
proportion = sqrtf(_initialFaceRect.size.area() / (float)_faceRect.size.area()); proportion = sqrtf(_initialFaceRect.size.area() / (float)_faceRect.size.area());
const float INITIAL_DISTANCE_TO_CAMERA = 0.333f; const float INITIAL_DISTANCE_TO_CAMERA = 0.333f;
z = INITIAL_DISTANCE_TO_CAMERA * proportion - INITIAL_DISTANCE_TO_CAMERA; z = INITIAL_DISTANCE_TO_CAMERA * proportion - INITIAL_DISTANCE_TO_CAMERA;
} else { } else {
z = (midFaceDepth - _initialFaceDepth) * METERS_PER_MM; z = (midFaceDepth - _initialFaceDepth) * METERS_PER_MM;
proportion = midFaceDepth / _initialFaceDepth; proportion = midFaceDepth / _initialFaceDepth;
} }
const float POSITION_SCALE = 0.5f; const float POSITION_SCALE = 0.5f;
@ -265,10 +265,10 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF
z); z);
} }
} }
// note that we have data // note that we have data
_active = true; _active = true;
// let the grabber know we're ready for the next frame // let the grabber know we're ready for the next frame
QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame())); QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame()));
} }
@ -289,21 +289,21 @@ static AvatarJointID xnToAvatarJoint(XnSkeletonJoint joint) {
case XN_SKEL_HEAD: return AVATAR_JOINT_HEAD_TOP; case XN_SKEL_HEAD: return AVATAR_JOINT_HEAD_TOP;
case XN_SKEL_NECK: return AVATAR_JOINT_HEAD_BASE; case XN_SKEL_NECK: return AVATAR_JOINT_HEAD_BASE;
case XN_SKEL_TORSO: return AVATAR_JOINT_CHEST; case XN_SKEL_TORSO: return AVATAR_JOINT_CHEST;
case XN_SKEL_LEFT_SHOULDER: return AVATAR_JOINT_RIGHT_ELBOW; case XN_SKEL_LEFT_SHOULDER: return AVATAR_JOINT_RIGHT_ELBOW;
case XN_SKEL_LEFT_ELBOW: return AVATAR_JOINT_RIGHT_WRIST; case XN_SKEL_LEFT_ELBOW: return AVATAR_JOINT_RIGHT_WRIST;
case XN_SKEL_RIGHT_SHOULDER: return AVATAR_JOINT_LEFT_ELBOW; case XN_SKEL_RIGHT_SHOULDER: return AVATAR_JOINT_LEFT_ELBOW;
case XN_SKEL_RIGHT_ELBOW: return AVATAR_JOINT_LEFT_WRIST; case XN_SKEL_RIGHT_ELBOW: return AVATAR_JOINT_LEFT_WRIST;
case XN_SKEL_LEFT_HIP: return AVATAR_JOINT_RIGHT_KNEE; case XN_SKEL_LEFT_HIP: return AVATAR_JOINT_RIGHT_KNEE;
case XN_SKEL_LEFT_KNEE: return AVATAR_JOINT_RIGHT_HEEL; case XN_SKEL_LEFT_KNEE: return AVATAR_JOINT_RIGHT_HEEL;
case XN_SKEL_LEFT_FOOT: return AVATAR_JOINT_RIGHT_TOES; case XN_SKEL_LEFT_FOOT: return AVATAR_JOINT_RIGHT_TOES;
case XN_SKEL_RIGHT_HIP: return AVATAR_JOINT_LEFT_KNEE; case XN_SKEL_RIGHT_HIP: return AVATAR_JOINT_LEFT_KNEE;
case XN_SKEL_RIGHT_KNEE: return AVATAR_JOINT_LEFT_HEEL; case XN_SKEL_RIGHT_KNEE: return AVATAR_JOINT_LEFT_HEEL;
case XN_SKEL_RIGHT_FOOT: return AVATAR_JOINT_LEFT_TOES; case XN_SKEL_RIGHT_FOOT: return AVATAR_JOINT_LEFT_TOES;
default: return AVATAR_JOINT_NULL; default: return AVATAR_JOINT_NULL;
} }
} }
@ -312,19 +312,19 @@ static int getParentJoint(XnSkeletonJoint joint) {
switch (joint) { switch (joint) {
case XN_SKEL_HEAD: return XN_SKEL_NECK; case XN_SKEL_HEAD: return XN_SKEL_NECK;
case XN_SKEL_TORSO: return -1; case XN_SKEL_TORSO: return -1;
case XN_SKEL_LEFT_ELBOW: return XN_SKEL_LEFT_SHOULDER; case XN_SKEL_LEFT_ELBOW: return XN_SKEL_LEFT_SHOULDER;
case XN_SKEL_LEFT_HAND: return XN_SKEL_LEFT_ELBOW; case XN_SKEL_LEFT_HAND: return XN_SKEL_LEFT_ELBOW;
case XN_SKEL_RIGHT_ELBOW: return XN_SKEL_RIGHT_SHOULDER; case XN_SKEL_RIGHT_ELBOW: return XN_SKEL_RIGHT_SHOULDER;
case XN_SKEL_RIGHT_HAND: return XN_SKEL_RIGHT_ELBOW; case XN_SKEL_RIGHT_HAND: return XN_SKEL_RIGHT_ELBOW;
case XN_SKEL_LEFT_KNEE: return XN_SKEL_LEFT_HIP; case XN_SKEL_LEFT_KNEE: return XN_SKEL_LEFT_HIP;
case XN_SKEL_LEFT_FOOT: return XN_SKEL_LEFT_KNEE; case XN_SKEL_LEFT_FOOT: return XN_SKEL_LEFT_KNEE;
case XN_SKEL_RIGHT_KNEE: return XN_SKEL_RIGHT_HIP; case XN_SKEL_RIGHT_KNEE: return XN_SKEL_RIGHT_HIP;
case XN_SKEL_RIGHT_FOOT: return XN_SKEL_RIGHT_KNEE; case XN_SKEL_RIGHT_FOOT: return XN_SKEL_RIGHT_KNEE;
default: return XN_SKEL_TORSO; default: return XN_SKEL_TORSO;
} }
} }
@ -359,7 +359,7 @@ static void XN_CALLBACK_TYPE calibrationCompleted(SkeletonCapability& capability
if (status == XN_CALIBRATION_STATUS_OK) { if (status == XN_CALIBRATION_STATUS_OK) {
qDebug("Calibration completed for user %d.\n", id); qDebug("Calibration completed for user %d.\n", id);
capability.StartTracking(id); capability.StartTracking(id);
} else { } else {
qDebug("Calibration failed to user %d.\n", id); qDebug("Calibration failed to user %d.\n", id);
capability.RequestCalibration(id, true); capability.RequestCalibration(id, true);
@ -370,7 +370,7 @@ static void XN_CALLBACK_TYPE calibrationCompleted(SkeletonCapability& capability
void FrameGrabber::cycleVideoSendMode() { void FrameGrabber::cycleVideoSendMode() {
_videoSendMode = (VideoSendMode)((_videoSendMode + 1) % VIDEO_SEND_MODE_COUNT); _videoSendMode = (VideoSendMode)((_videoSendMode + 1) % VIDEO_SEND_MODE_COUNT);
_searchWindow = cv::Rect(0, 0, 0, 0); _searchWindow = cv::Rect(0, 0, 0, 0);
destroyCodecs(); destroyCodecs();
} }
@ -391,7 +391,7 @@ void FrameGrabber::shutdown() {
} }
destroyCodecs(); destroyCodecs();
_initialized = false; _initialized = false;
thread()->quit(); thread()->quit();
} }
@ -407,17 +407,17 @@ void FrameGrabber::grabFrame() {
int format = GL_BGR; int format = GL_BGR;
Mat color, depth; Mat color, depth;
JointVector joints; JointVector joints;
#ifdef HAVE_OPENNI #ifdef HAVE_OPENNI
if (_depthGenerator.IsValid()) { if (_depthGenerator.IsValid()) {
_xnContext.WaitAnyUpdateAll(); _xnContext.WaitAnyUpdateAll();
color = Mat(_imageMetaData.YRes(), _imageMetaData.XRes(), CV_8UC3, (void*)_imageGenerator.GetImageMap()); color = Mat(_imageMetaData.YRes(), _imageMetaData.XRes(), CV_8UC3, (void*)_imageGenerator.GetImageMap());
format = GL_RGB; format = GL_RGB;
depth = Mat(_depthMetaData.YRes(), _depthMetaData.XRes(), CV_16UC1, (void*)_depthGenerator.GetDepthMap()); depth = Mat(_depthMetaData.YRes(), _depthMetaData.XRes(), CV_16UC1, (void*)_depthGenerator.GetDepthMap());
_userID = 0; _userID = 0;
XnUInt16 userCount = 1; XnUInt16 userCount = 1;
_userGenerator.GetUsers(&_userID, userCount); _userGenerator.GetUsers(&_userID, userCount);
if (userCount > 0 && _userGenerator.GetSkeletonCap().IsTracking(_userID)) { if (userCount > 0 && _userGenerator.GetSkeletonCap().IsTracking(_userID)) {
joints.resize(NUM_AVATAR_JOINTS); joints.resize(NUM_AVATAR_JOINTS);
@ -464,7 +464,7 @@ void FrameGrabber::grabFrame() {
} }
color = image; color = image;
} }
const int ENCODED_FACE_WIDTH = 128; const int ENCODED_FACE_WIDTH = 128;
const int ENCODED_FACE_HEIGHT = 128; const int ENCODED_FACE_HEIGHT = 128;
int encodedWidth; int encodedWidth;
@ -480,7 +480,7 @@ void FrameGrabber::grabFrame() {
encodedHeight = color.rows; encodedHeight = color.rows;
aspectRatio = FULL_FRAME_ASPECT; aspectRatio = FULL_FRAME_ASPECT;
colorBitrateMultiplier = 4.0f; colorBitrateMultiplier = 4.0f;
} else { } else {
// if we don't have a search window (yet), try using the face cascade // if we don't have a search window (yet), try using the face cascade
int channels = 0; int channels = 0;
@ -492,7 +492,7 @@ void FrameGrabber::grabFrame() {
if (!faces.empty()) { if (!faces.empty()) {
_searchWindow = faces.front(); _searchWindow = faces.front();
updateHSVFrame(color, format); updateHSVFrame(color, format);
Mat faceHsv(_hsvFrame, _searchWindow); Mat faceHsv(_hsvFrame, _searchWindow);
Mat faceMask(_mask, _searchWindow); Mat faceMask(_mask, _searchWindow);
int sizes = 30; int sizes = 30;
@ -505,10 +505,10 @@ void FrameGrabber::grabFrame() {
RotatedRect faceRect; RotatedRect faceRect;
if (_searchWindow.area() > 0) { if (_searchWindow.area() > 0) {
updateHSVFrame(color, format); updateHSVFrame(color, format);
calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range); calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range);
bitwise_and(_backProject, _mask, _backProject); bitwise_and(_backProject, _mask, _backProject);
faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1)); faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1));
Rect faceBounds = faceRect.boundingRect(); Rect faceBounds = faceRect.boundingRect();
Rect imageBounds(0, 0, color.cols, color.rows); Rect imageBounds(0, 0, color.cols, color.rows);
@ -517,28 +517,28 @@ void FrameGrabber::grabFrame() {
encodedWidth = ENCODED_FACE_WIDTH; encodedWidth = ENCODED_FACE_WIDTH;
encodedHeight = ENCODED_FACE_HEIGHT; encodedHeight = ENCODED_FACE_HEIGHT;
depthBitrateMultiplier = 2.0f; depthBitrateMultiplier = 2.0f;
// correct for 180 degree rotations // correct for 180 degree rotations
if (faceRect.angle < -90.0f) { if (faceRect.angle < -90.0f) {
faceRect.angle += 180.0f; faceRect.angle += 180.0f;
} else if (faceRect.angle > 90.0f) { } else if (faceRect.angle > 90.0f) {
faceRect.angle -= 180.0f; faceRect.angle -= 180.0f;
} }
// compute the smoothed face rect // compute the smoothed face rect
if (_smoothedFaceRect.size.area() == 0) { if (_smoothedFaceRect.size.area() == 0) {
_smoothedFaceRect = faceRect; _smoothedFaceRect = faceRect;
} else { } else {
const float FACE_RECT_SMOOTHING = 0.9f; const float FACE_RECT_SMOOTHING = 0.9f;
_smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING); _smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING);
_smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING); _smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING);
_smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING); _smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING);
_smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING); _smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING);
_smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING);
} }
// use the face rect to compute the face transform, aspect ratio // use the face rect to compute the face transform, aspect ratio
Point2f sourcePoints[4]; Point2f sourcePoints[4];
_smoothedFaceRect.points(sourcePoints); _smoothedFaceRect.points(sourcePoints);
@ -546,7 +546,7 @@ void FrameGrabber::grabFrame() {
faceTransform = getAffineTransform(sourcePoints, destPoints); faceTransform = getAffineTransform(sourcePoints, destPoints);
aspectRatio = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; aspectRatio = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height;
} }
const ushort ELEVEN_BIT_MINIMUM = 0; const ushort ELEVEN_BIT_MINIMUM = 0;
const uchar EIGHT_BIT_MIDPOINT = 128; const uchar EIGHT_BIT_MIDPOINT = 128;
double depthOffset; double depthOffset;
@ -555,12 +555,12 @@ void FrameGrabber::grabFrame() {
// warp the face depth without interpolation (because it will contain invalid zero values) // warp the face depth without interpolation (because it will contain invalid zero values)
_faceDepth.create(encodedHeight, encodedWidth, CV_16UC1); _faceDepth.create(encodedHeight, encodedWidth, CV_16UC1);
warpAffine(depth, _faceDepth, faceTransform, _faceDepth.size(), INTER_NEAREST); warpAffine(depth, _faceDepth, faceTransform, _faceDepth.size(), INTER_NEAREST);
} else { } else {
_faceDepth = depth; _faceDepth = depth;
} }
_smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1); _smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1);
// smooth the depth over time // smooth the depth over time
const ushort ELEVEN_BIT_MAXIMUM = 2047; const ushort ELEVEN_BIT_MAXIMUM = 2047;
const float DEPTH_SMOOTHING = 0.25f; const float DEPTH_SMOOTHING = 0.25f;
@ -580,7 +580,7 @@ void FrameGrabber::grabFrame() {
const ushort MINIMUM_DEPTH_OFFSET = 64; const ushort MINIMUM_DEPTH_OFFSET = 64;
const float FIXED_MID_DEPTH = 640.0f; const float FIXED_MID_DEPTH = 640.0f;
float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH; float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH;
// smooth the mid face depth over time // smooth the mid face depth over time
const float MID_FACE_DEPTH_SMOOTHING = 0.5f; const float MID_FACE_DEPTH_SMOOTHING = 0.5f;
_smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth :
@ -590,35 +590,35 @@ void FrameGrabber::grabFrame() {
depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth;
depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset);
} }
QByteArray payload; QByteArray payload;
if (_videoSendMode != NO_VIDEO) { if (_videoSendMode != NO_VIDEO) {
if (_colorCodec.name == 0) { if (_colorCodec.name == 0) {
// initialize encoder context(s) // initialize encoder context(s)
vpx_codec_enc_cfg_t codecConfig; vpx_codec_enc_cfg_t codecConfig;
vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0); vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0);
codecConfig.rc_target_bitrate = ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * colorBitrateMultiplier * codecConfig.rc_target_bitrate = ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * colorBitrateMultiplier *
codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h; codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h;
codecConfig.g_w = encodedWidth; codecConfig.g_w = encodedWidth;
codecConfig.g_h = encodedHeight; codecConfig.g_h = encodedHeight;
vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0); vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0);
if (!depth.empty()) { if (!depth.empty()) {
codecConfig.rc_target_bitrate *= depthBitrateMultiplier; codecConfig.rc_target_bitrate *= depthBitrateMultiplier;
vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0); vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0);
} }
} }
Mat transform; Mat transform;
if (_videoSendMode == FACE_VIDEO) { if (_videoSendMode == FACE_VIDEO) {
// resize/rotate face into encoding rectangle // resize/rotate face into encoding rectangle
_faceColor.create(encodedHeight, encodedWidth, CV_8UC3); _faceColor.create(encodedHeight, encodedWidth, CV_8UC3);
warpAffine(color, _faceColor, faceTransform, _faceColor.size()); warpAffine(color, _faceColor, faceTransform, _faceColor.size());
} else { } else {
_faceColor = color; _faceColor = color;
} }
// convert from RGB to YV12: see http://www.fourcc.org/yuv.php and // convert from RGB to YV12: see http://www.fourcc.org/yuv.php and
// http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor
const int ENCODED_BITS_PER_Y = 8; const int ENCODED_BITS_PER_Y = 8;
@ -653,7 +653,7 @@ void FrameGrabber::grabFrame() {
uchar* tr = _faceColor.ptr(i, j + 1); uchar* tr = _faceColor.ptr(i, j + 1);
uchar* bl = _faceColor.ptr(i + 1, j); uchar* bl = _faceColor.ptr(i + 1, j);
uchar* br = _faceColor.ptr(i + 1, j + 1); uchar* br = _faceColor.ptr(i + 1, j + 1);
ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8; ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8;
ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8; ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8;
ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] * ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] *
@ -661,12 +661,12 @@ void FrameGrabber::grabFrame() {
ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] * ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] *
Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8; Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8;
ydest += 2; ydest += 2;
int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex]; int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex];
int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex]; int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex];
int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex]; int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex];
int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8; int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8;
*vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128; *vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128;
*udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128; *udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128;
} }
@ -674,7 +674,7 @@ void FrameGrabber::grabFrame() {
vline += vpxImage.stride[1]; vline += vpxImage.stride[1];
uline += vpxImage.stride[2]; uline += vpxImage.stride[2];
} }
// encode the frame // encode the frame
vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME);
@ -691,7 +691,7 @@ void FrameGrabber::grabFrame() {
payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz);
} }
} }
if (!depth.empty()) { if (!depth.empty()) {
// convert with mask // convert with mask
uchar* yline = vpxImage.planes[0]; uchar* yline = vpxImage.planes[0];
@ -707,9 +707,9 @@ void FrameGrabber::grabFrame() {
ushort tr = *_smoothedFaceDepth.ptr<ushort>(i, j + 1); ushort tr = *_smoothedFaceDepth.ptr<ushort>(i, j + 1);
ushort bl = *_smoothedFaceDepth.ptr<ushort>(i + 1, j); ushort bl = *_smoothedFaceDepth.ptr<ushort>(i + 1, j);
ushort br = *_smoothedFaceDepth.ptr<ushort>(i + 1, j + 1); ushort br = *_smoothedFaceDepth.ptr<ushort>(i + 1, j + 1);
uchar mask = EIGHT_BIT_MAXIMUM; uchar mask = EIGHT_BIT_MAXIMUM;
ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) :
saturate_cast<uchar>(tl + depthOffset); saturate_cast<uchar>(tl + depthOffset);
ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) :
@ -719,7 +719,7 @@ void FrameGrabber::grabFrame() {
ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) :
saturate_cast<uchar>(br + depthOffset); saturate_cast<uchar>(br + depthOffset);
ydest += 2; ydest += 2;
*vdest++ = mask; *vdest++ = mask;
*udest++ = EIGHT_BIT_MIDPOINT; *udest++ = EIGHT_BIT_MIDPOINT;
} }
@ -727,7 +727,7 @@ void FrameGrabber::grabFrame() {
vline += vpxImage.stride[1]; vline += vpxImage.stride[1];
uline += vpxImage.stride[2]; uline += vpxImage.stride[2];
} }
// encode the frame // encode the frame
vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME); vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME);
@ -741,10 +741,10 @@ void FrameGrabber::grabFrame() {
} }
} }
} }
QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage",
Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload));
QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame",
Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth),
Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(bool, !payload.isEmpty()), Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(bool, !payload.isEmpty()),
@ -770,19 +770,19 @@ bool FrameGrabber::init() {
_depthGenerator.GetMetaData(_depthMetaData); _depthGenerator.GetMetaData(_depthMetaData);
_imageGenerator.SetPixelFormat(XN_PIXEL_FORMAT_RGB24); _imageGenerator.SetPixelFormat(XN_PIXEL_FORMAT_RGB24);
_imageGenerator.GetMetaData(_imageMetaData); _imageGenerator.GetMetaData(_imageMetaData);
XnCallbackHandle userCallbacks, calibrationStartCallback, calibrationCompleteCallback; XnCallbackHandle userCallbacks, calibrationStartCallback, calibrationCompleteCallback;
_userGenerator.RegisterUserCallbacks(newUser, lostUser, 0, userCallbacks); _userGenerator.RegisterUserCallbacks(newUser, lostUser, 0, userCallbacks);
_userGenerator.GetSkeletonCap().RegisterToCalibrationStart(calibrationStarted, 0, calibrationStartCallback); _userGenerator.GetSkeletonCap().RegisterToCalibrationStart(calibrationStarted, 0, calibrationStartCallback);
_userGenerator.GetSkeletonCap().RegisterToCalibrationComplete(calibrationCompleted, 0, calibrationCompleteCallback); _userGenerator.GetSkeletonCap().RegisterToCalibrationComplete(calibrationCompleted, 0, calibrationCompleteCallback);
_userGenerator.GetSkeletonCap().SetSkeletonProfile(XN_SKEL_PROFILE_UPPER); _userGenerator.GetSkeletonCap().SetSkeletonProfile(XN_SKEL_PROFILE_UPPER);
// make the depth viewpoint match that of the video image // make the depth viewpoint match that of the video image
if (_depthGenerator.IsCapabilitySupported(XN_CAPABILITY_ALTERNATIVE_VIEW_POINT)) { if (_depthGenerator.IsCapabilitySupported(XN_CAPABILITY_ALTERNATIVE_VIEW_POINT)) {
_depthGenerator.GetAlternativeViewPointCap().SetViewPoint(_imageGenerator); _depthGenerator.GetAlternativeViewPointCap().SetViewPoint(_imageGenerator);
} }
_xnContext.StartGeneratingAll(); _xnContext.StartGeneratingAll();
return true; return true;
} }
@ -797,7 +797,7 @@ bool FrameGrabber::init() {
const int IDEAL_FRAME_HEIGHT = 240; const int IDEAL_FRAME_HEIGHT = 240;
cvSetCaptureProperty(_capture, CV_CAP_PROP_FRAME_WIDTH, IDEAL_FRAME_WIDTH); cvSetCaptureProperty(_capture, CV_CAP_PROP_FRAME_WIDTH, IDEAL_FRAME_WIDTH);
cvSetCaptureProperty(_capture, CV_CAP_PROP_FRAME_HEIGHT, IDEAL_FRAME_HEIGHT); cvSetCaptureProperty(_capture, CV_CAP_PROP_FRAME_HEIGHT, IDEAL_FRAME_HEIGHT);
#ifdef __APPLE__ #ifdef __APPLE__
configureCamera(0x5ac, 0x8510, false, 0.975, 0.5, 1.0, 0.5, true, 0.5); configureCamera(0x5ac, 0x8510, false, 0.975, 0.5, 1.0, 0.5, true, 0.5);
#else #else

48
interface/src/Webcam.h
View file

@ -38,9 +38,9 @@ typedef QVector<Joint> JointVector;
class Webcam : public QObject { class Webcam : public QObject {
Q_OBJECT Q_OBJECT
public: public:
Webcam(); Webcam();
~Webcam(); ~Webcam();
@ -49,34 +49,34 @@ public:
bool isActive() const { return _active; } bool isActive() const { return _active; }
bool isSending() const { return _sending; } bool isSending() const { return _sending; }
GLuint getColorTextureID() const { return _colorTextureID; } GLuint getColorTextureID() const { return _colorTextureID; }
GLuint getDepthTextureID() const { return _depthTextureID; } GLuint getDepthTextureID() const { return _depthTextureID; }
const cv::Size2f& getTextureSize() const { return _textureSize; } const cv::Size2f& getTextureSize() const { return _textureSize; }
float getAspectRatio() const { return _aspectRatio; } float getAspectRatio() const { return _aspectRatio; }
const cv::RotatedRect& getFaceRect() const { return _faceRect; } const cv::RotatedRect& getFaceRect() const { return _faceRect; }
const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; } const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; }
const glm::vec3& getEstimatedRotation() const { return _estimatedRotation; } const glm::vec3& getEstimatedRotation() const { return _estimatedRotation; }
const JointVector& getEstimatedJoints() const { return _estimatedJoints; } const JointVector& getEstimatedJoints() const { return _estimatedJoints; }
void reset(); void reset();
void renderPreview(int screenWidth, int screenHeight); void renderPreview(int screenWidth, int screenHeight);
public slots: public slots:
void setEnabled(bool enabled); void setEnabled(bool enabled);
void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth, void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth,
float aspectRatio, const cv::RotatedRect& faceRect, bool sending, const JointVector& joints); float aspectRatio, const cv::RotatedRect& faceRect, bool sending, const JointVector& joints);
void setSkeletonTrackingOn(bool toggle) { _skeletonTrackingOn = toggle; }; void setSkeletonTrackingOn(bool toggle) { _skeletonTrackingOn = toggle; };
private: private:
QThread _grabberThread; QThread _grabberThread;
FrameGrabber* _grabber; FrameGrabber* _grabber;
bool _enabled; bool _enabled;
bool _active; bool _active;
bool _sending; bool _sending;
@ -88,12 +88,12 @@ private:
cv::RotatedRect _initialFaceRect; cv::RotatedRect _initialFaceRect;
float _initialFaceDepth; float _initialFaceDepth;
JointVector _joints; JointVector _joints;
uint64_t _startTimestamp; uint64_t _startTimestamp;
int _frameCount; int _frameCount;
uint64_t _lastFrameTimestamp; uint64_t _lastFrameTimestamp;
glm::vec3 _estimatedPosition; glm::vec3 _estimatedPosition;
glm::vec3 _estimatedRotation; glm::vec3 _estimatedRotation;
JointVector _estimatedJoints; JointVector _estimatedJoints;
@ -103,27 +103,27 @@ private:
class FrameGrabber : public QObject { class FrameGrabber : public QObject {
Q_OBJECT Q_OBJECT
public: public:
FrameGrabber(); FrameGrabber();
virtual ~FrameGrabber(); virtual ~FrameGrabber();
public slots: public slots:
void cycleVideoSendMode(); void cycleVideoSendMode();
void reset(); void reset();
void shutdown(); void shutdown();
void grabFrame(); void grabFrame();
private: private:
enum VideoSendMode { NO_VIDEO, FACE_VIDEO, FULL_FRAME_VIDEO, VIDEO_SEND_MODE_COUNT }; enum VideoSendMode { NO_VIDEO, FACE_VIDEO, FULL_FRAME_VIDEO, VIDEO_SEND_MODE_COUNT };
bool init(); bool init();
void updateHSVFrame(const cv::Mat& frame, int format); void updateHSVFrame(const cv::Mat& frame, int format);
void destroyCodecs(); void destroyCodecs();
bool _initialized; bool _initialized;
VideoSendMode _videoSendMode; VideoSendMode _videoSendMode;
CvCapture* _capture; CvCapture* _capture;
@ -135,7 +135,7 @@ private:
cv::Rect _searchWindow; cv::Rect _searchWindow;
cv::Mat _grayDepthFrame; cv::Mat _grayDepthFrame;
float _smoothedMidFaceDepth; float _smoothedMidFaceDepth;
vpx_codec_ctx_t _colorCodec; vpx_codec_ctx_t _colorCodec;
vpx_codec_ctx_t _depthCodec; vpx_codec_ctx_t _depthCodec;
int _frameCount; int _frameCount;
@ -144,7 +144,7 @@ private:
cv::Mat _smoothedFaceDepth; cv::Mat _smoothedFaceDepth;
QByteArray _encodedFace; QByteArray _encodedFace;
cv::RotatedRect _smoothedFaceRect; cv::RotatedRect _smoothedFaceRect;
#ifdef HAVE_OPENNI #ifdef HAVE_OPENNI
xn::Context _xnContext; xn::Context _xnContext;
xn::DepthGenerator _depthGenerator; xn::DepthGenerator _depthGenerator;
@ -158,10 +158,10 @@ private:
class Joint { class Joint {
public: public:
Joint(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& projected); Joint(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& projected);
Joint(); Joint();
bool isValid; bool isValid;
glm::vec3 position; glm::vec3 position;
glm::quat rotation; glm::quat rotation;