From 401ef45000e49113d18f55a2c9df215d0961a020 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 16 Jul 2013 17:56:53 -0700 Subject: [PATCH] Removed keypoint stuff, fixed stupidity leading to fringed edges (was linearly interpolating the missing values). --- interface/src/Webcam.cpp | 51 ++++++++------------------------- interface/src/Webcam.h | 9 ++---- interface/src/avatar/Avatar.cpp | 1 - interface/src/avatar/Face.cpp | 21 ++------------ interface/src/avatar/Face.h | 5 +--- 5 files changed, 17 insertions(+), 70 deletions(-) diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index a053f7e3e4..b00569dc11 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -27,7 +27,6 @@ using namespace xn; // register types with Qt metatype system int jointVectorMetaType = qRegisterMetaType("JointVector"); -int keyPointVectorMetaType = qRegisterMetaType("KeyPointVector"); int matMetaType = qRegisterMetaType("cv::Mat"); int rotatedRectMetaType = qRegisterMetaType("cv::RotatedRect"); @@ -133,18 +132,8 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) { glVertex2f(left + facePoints[3].x * xScale, top + facePoints[3].y * yScale); glEnd(); - if (!_keyPoints.empty()) { - glColor3f(0.0f, 1.0f, 0.0f); - glBegin(GL_POINTS); - for (KeyPointVector::iterator it = _keyPoints.begin(); it != _keyPoints.end(); it++) { - glVertex2f(left + it->pt.x * xScale, top + it->pt.y * yScale); - } - glEnd(); - } - char fps[30]; - sprintf(fps, "FPS: %d, Points: %d", (int)(roundf(_frameCount * 1000000.0f / (usecTimestampNow() - _startTimestamp))), - (int)_keyPoints.size()); + sprintf(fps, "FPS: %d", (int)(roundf(_frameCount * 1000000.0f / (usecTimestampNow() - _startTimestamp)))); drawtext(left, top + PREVIEW_HEIGHT + 20, 0.10, 0, 1, 0, fps); } } @@ -157,8 +146,7 @@ Webcam::~Webcam() { delete _grabber; } -void Webcam::setFrame(const Mat& color, int format, const Mat& depth, const Mat& depthPreview, - const RotatedRect& faceRect, const KeyPointVector& keyPoints, const JointVector& joints) { +void Webcam::setFrame(const Mat& color, int format, const Mat& depth, const RotatedRect& faceRect, const JointVector& joints) { IplImage colorImage = color; glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); if (_colorTextureID == 0) { @@ -175,15 +163,16 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, const Mat& GL_UNSIGNED_BYTE, colorImage.imageData); } - if (!depthPreview.empty()) { - IplImage depthImage = depthPreview; + if (!depth.empty()) { + IplImage depthImage = depth; glPixelStorei(GL_UNPACK_ROW_LENGTH, depthImage.widthStep); if (_depthTextureID == 0) { glGenTextures(1, &_depthTextureID); glBindTexture(GL_TEXTURE_2D, _depthTextureID); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depthImage.width, depthImage.height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, depthImage.imageData); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); } else { glBindTexture(GL_TEXTURE_2D, _depthTextureID); @@ -197,7 +186,6 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, const Mat& // store our face rect and joints, update our frame count for fps computation _faceRect = faceRect; _joints = joints; - _keyPoints = keyPoints; _frameCount++; const int MAX_FPS = 60; @@ -288,7 +276,7 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, const Mat& QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame())); } -FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), _depthOffset(-512.0) { +FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), _depthOffset(0.0) { } FrameGrabber::~FrameGrabber() { @@ -501,7 +489,7 @@ void FrameGrabber::grabFrame() { faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1)); Rect faceBounds = faceRect.boundingRect(); - Rect imageBounds(0, 0, depth.cols, depth.rows); + Rect imageBounds(0, 0, color.cols, color.rows); _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); } @@ -509,31 +497,16 @@ void FrameGrabber::grabFrame() { if (_depthGenerator.IsValid()) { if (_searchWindow.area() > 0) { const double DEPTH_OFFSET_SMOOTHING = 0.95; - _depthOffset = glm::mix(128.0 - mean(depth(_searchWindow))[0], _depthOffset, DEPTH_OFFSET_SMOOTHING); + double meanOffset = 128.0 - mean(depth(_searchWindow))[0]; + _depthOffset = (_depthOffset == 0.0) ? meanOffset : glm::mix(meanOffset, _depthOffset, DEPTH_OFFSET_SMOOTHING); } depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, _depthOffset); } #endif - KeyPointVector keyPoints; - if (!_hsvFrame.empty()) { - _grayFrame.create(_hsvFrame.rows, _hsvFrame.cols, CV_8UC1); - int fromTo[] = { 2, 0 }; - Mat hsvInner = _hsvFrame(_searchWindow); - Mat grayInner = _grayFrame(_searchWindow); - mixChannels(&hsvInner, 1, &grayInner, 1, fromTo, 1); - FAST(grayInner, keyPoints, 4); - - // offset the detected points - for (KeyPointVector::iterator it = keyPoints.begin(); it != keyPoints.end(); it++) { - it->pt.x += _searchWindow.x; - it->pt.y += _searchWindow.y; - } - } - QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", - Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, depth), Q_ARG(cv::Mat, _grayDepthFrame), - Q_ARG(cv::RotatedRect, faceRect), Q_ARG(KeyPointVector, keyPoints), Q_ARG(JointVector, joints)); + Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), + Q_ARG(cv::RotatedRect, faceRect), Q_ARG(JointVector, joints)); } bool FrameGrabber::init() { diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index cc22ecf602..34ed444e4b 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -33,7 +33,6 @@ class FrameGrabber; class Joint; typedef QVector JointVector; -typedef std::vector KeyPointVector; class Webcam : public QObject { Q_OBJECT @@ -48,7 +47,6 @@ public: GLuint getColorTextureID() const { return _colorTextureID; } GLuint getDepthTextureID() const { return _depthTextureID; } const cv::Size2f& getTextureSize() const { return _textureSize; } - const KeyPointVector& getKeyPoints() const { return _keyPoints; } const cv::RotatedRect& getEstimatedFaceRect() const { return _estimatedFaceRect; } const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; } @@ -61,8 +59,8 @@ public: public slots: void setEnabled(bool enabled); - void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, const cv::Mat& depthPreview, - const cv::RotatedRect& faceRect, const KeyPointVector& keyPoints, const JointVector& joints); + void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, + const cv::RotatedRect& faceRect, const JointVector& joints); private: @@ -76,7 +74,6 @@ private: cv::Size2f _textureSize; cv::RotatedRect _faceRect; cv::RotatedRect _initialFaceRect; - KeyPointVector _keyPoints; JointVector _joints; uint64_t _startTimestamp; @@ -113,7 +110,6 @@ private: CvCapture* _capture; cv::CascadeClassifier _faceCascade; cv::Mat _hsvFrame; - cv::Mat _grayFrame; cv::Mat _mask; cv::SparseMat _histogram; cv::Mat _backProject; @@ -145,7 +141,6 @@ public: }; Q_DECLARE_METATYPE(JointVector) -Q_DECLARE_METATYPE(KeyPointVector) Q_DECLARE_METATYPE(cv::Mat) Q_DECLARE_METATYPE(cv::RotatedRect) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 9752fe2a45..57bc19933a 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -316,7 +316,6 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook, _head.getFace().setDepthTextureID(webcam->getDepthTextureID()); _head.getFace().setTextureSize(webcam->getTextureSize()); _head.getFace().setTextureRect(webcam->getEstimatedFaceRect()); - _head.getFace().setKeyPoints(webcam->getKeyPoints()); // compute and store the joint rotations const JointVector& joints = webcam->getEstimatedJoints(); diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp index b2d97616f1..fdcc8624ed 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/Face.cpp @@ -22,7 +22,7 @@ int Face::_texCoordUpLocation; GLuint Face::_vboID; GLuint Face::_iboID; -Face::Face(Head* owningHead) : _owningHead(owningHead), _renderMode(POINTS), _colorTextureID(0), _depthTextureID(0) { +Face::Face(Head* owningHead) : _owningHead(owningHead), _renderMode(MESH), _colorTextureID(0), _depthTextureID(0) { } bool Face::render(float alpha) { @@ -130,27 +130,10 @@ bool Face::render(float alpha) { glDrawRangeElementsEXT(GL_TRIANGLES, 0, VERTEX_COUNT - 1, INDEX_COUNT, GL_UNSIGNED_INT, 0); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); - } else if (_renderMode == POINTS) { + } else { // _renderMode == POINTS glPointSize(5.0f); glDrawArrays(GL_POINTS, 0, VERTEX_COUNT); glPointSize(1.0f); - - } else if (_renderMode == KEY_POINTS) { - glPointSize(10.0f); - float widthSquared = _textureRect.size.width * _textureRect.size.width; - float heightSquared = _textureRect.size.height * _textureRect.size.height; - float rightX = (points[3].x - points[0].x) / widthSquared; - float rightY = (points[3].y - points[0].y) / widthSquared; - float upX = (points[1].x - points[0].x) / heightSquared; - float upY = (points[1].y - points[0].y) / heightSquared; - glBegin(GL_POINTS); - for (KeyPointVector::iterator it = _keyPoints.begin(); it != _keyPoints.end(); it++) { - float relativeX = it->pt.x - points[0].x; - float relativeY = it->pt.y - points[0].y; - glVertex2f(relativeX * rightX + relativeY * rightY, relativeX * upX + relativeY * upY); - } - glEnd(); - glPointSize(1.0f); } glDisable(GL_ALPHA_TEST); diff --git a/interface/src/avatar/Face.h b/interface/src/avatar/Face.h index 326b447648..58fbf59a78 100644 --- a/interface/src/avatar/Face.h +++ b/interface/src/avatar/Face.h @@ -14,7 +14,6 @@ #include #include "InterfaceConfig.h" -#include "Webcam.h" class Head; class ProgramObject; @@ -30,7 +29,6 @@ public: void setDepthTextureID(GLuint depthTextureID) { _depthTextureID = depthTextureID; } void setTextureSize(const cv::Size2f& textureSize) { _textureSize = textureSize; } void setTextureRect(const cv::RotatedRect& textureRect) { _textureRect = textureRect; } - void setKeyPoints(const KeyPointVector& keyPoints) { _keyPoints = keyPoints; } bool render(float alpha); @@ -40,7 +38,7 @@ public slots: private: - enum RenderMode { MESH, POINTS, KEY_POINTS, RENDER_MODE_COUNT }; + enum RenderMode { MESH, POINTS, RENDER_MODE_COUNT }; Head* _owningHead; RenderMode _renderMode; @@ -48,7 +46,6 @@ private: GLuint _depthTextureID; cv::Size2f _textureSize; cv::RotatedRect _textureRect; - KeyPointVector _keyPoints; static ProgramObject* _program; static int _texCoordCornerLocation;