From 729158f8824c8002921f23fd2e5584e45f66b968 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 31 Jul 2013 12:19:14 -0700 Subject: [PATCH 01/14] Added depth smoothing, experimenting with using depth minimum rather than mean. --- interface/src/Webcam.cpp | 30 ++++++++++++++++++++---------- interface/src/Webcam.h | 1 + 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index d4fa015ba0..032b483082 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -620,25 +620,35 @@ void FrameGrabber::grabFrame() { _faceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST); - // find the mean of the valid values - qint64 depthTotal = 0; - qint64 depthSamples = 0; - ushort* src = _faceDepth.ptr(); + _smoothedFaceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); + + // smooth and find the minimum/mean of the valid values const ushort ELEVEN_BIT_MINIMUM = 0; const ushort ELEVEN_BIT_MAXIMUM = 2047; + const float DEPTH_SMOOTHING = 0.25f; + qint64 depthTotal = 0; + qint64 depthSamples = 0; + ushort depthMinimum = ELEVEN_BIT_MAXIMUM; + ushort* src = _faceDepth.ptr(); + ushort* dest = _smoothedFaceDepth.ptr(); for (int i = 0; i < ENCODED_FACE_HEIGHT; i++) { for (int j = 0; j < ENCODED_FACE_WIDTH; j++) { ushort depth = *src++; if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { depthTotal += depth; + depthMinimum = min(depthMinimum, depth); depthSamples++; + + *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); } + dest++; } } - float mean = (depthSamples == 0) ? UNINITIALIZED_FACE_DEPTH : depthTotal / (float)depthSamples; + const ushort DEPTH_MINIMUM_OFFSET = 64; + float mean = (depthSamples == 0) ? UNINITIALIZED_FACE_DEPTH : depthMinimum + DEPTH_MINIMUM_OFFSET; // smooth the mean over time - const float DEPTH_OFFSET_SMOOTHING = 0.95f; + const float DEPTH_OFFSET_SMOOTHING = 0.5f; _smoothedMeanFaceDepth = (_smoothedMeanFaceDepth == UNINITIALIZED_FACE_DEPTH) ? mean : glm::mix(mean, _smoothedMeanFaceDepth, DEPTH_OFFSET_SMOOTHING); @@ -657,10 +667,10 @@ void FrameGrabber::grabFrame() { uchar* vdest = vline; uchar* udest = uline; for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { - ushort tl = *_faceDepth.ptr(i, j); - ushort tr = *_faceDepth.ptr(i, j + 1); - ushort bl = *_faceDepth.ptr(i + 1, j); - ushort br = *_faceDepth.ptr(i + 1, j + 1); + ushort tl = *_smoothedFaceDepth.ptr(i, j); + ushort tr = *_smoothedFaceDepth.ptr(i, j + 1); + ushort bl = *_smoothedFaceDepth.ptr(i + 1, j); + ushort br = *_smoothedFaceDepth.ptr(i + 1, j + 1); uchar mask = EIGHT_BIT_MAXIMUM; diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index 6c6d250897..077e9eed98 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -125,6 +125,7 @@ private: int _frameCount; cv::Mat _faceColor; cv::Mat _faceDepth; + cv::Mat _smoothedFaceDepth; QByteArray _encodedFace; cv::RotatedRect _smoothedFaceRect; From 415949cc26551be3b2d7127c2bb0c6b471c8bb4c Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 31 Jul 2013 14:40:33 -0700 Subject: [PATCH 02/14] Terminology change to reflect the fact that we're no longer using the mean. --- interface/src/Webcam.cpp | 41 ++++++++++++++++++---------------------- interface/src/Webcam.h | 4 ++-- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index 032b483082..3457064733 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -154,7 +154,7 @@ Webcam::~Webcam() { const float METERS_PER_MM = 1.0f / 1000.0f; -void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float meanFaceDepth, +void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midFaceDepth, const RotatedRect& faceRect, const JointVector& joints) { IplImage colorImage = color; glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); @@ -242,18 +242,18 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean if (_initialFaceRect.size.area() == 0) { _initialFaceRect = _faceRect; _estimatedPosition = glm::vec3(); - _initialFaceDepth = meanFaceDepth; + _initialFaceDepth = midFaceDepth; } else { float proportion, z; - if (meanFaceDepth == UNINITIALIZED_FACE_DEPTH) { + if (midFaceDepth == UNINITIALIZED_FACE_DEPTH) { proportion = sqrtf(_initialFaceRect.size.area() / (float)_faceRect.size.area()); const float INITIAL_DISTANCE_TO_CAMERA = 0.333f; z = INITIAL_DISTANCE_TO_CAMERA * proportion - INITIAL_DISTANCE_TO_CAMERA; } else { - z = (meanFaceDepth - _initialFaceDepth) * METERS_PER_MM; - proportion = meanFaceDepth / _initialFaceDepth; + z = (midFaceDepth - _initialFaceDepth) * METERS_PER_MM; + proportion = midFaceDepth / _initialFaceDepth; } const float POSITION_SCALE = 0.5f; _estimatedPosition = glm::vec3( @@ -271,7 +271,7 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean } FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), - _smoothedMeanFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) { + _smoothedMidFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) { } FrameGrabber::~FrameGrabber() { @@ -494,7 +494,7 @@ void FrameGrabber::grabFrame() { Rect imageBounds(0, 0, color.cols, color.rows); _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); } - + const int ENCODED_FACE_WIDTH = 128; const int ENCODED_FACE_HEIGHT = 128; if (_colorCodec.name == 0) { @@ -622,39 +622,34 @@ void FrameGrabber::grabFrame() { _smoothedFaceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); - // smooth and find the minimum/mean of the valid values + // smooth the depth over time const ushort ELEVEN_BIT_MINIMUM = 0; const ushort ELEVEN_BIT_MAXIMUM = 2047; const float DEPTH_SMOOTHING = 0.25f; - qint64 depthTotal = 0; - qint64 depthSamples = 0; - ushort depthMinimum = ELEVEN_BIT_MAXIMUM; ushort* src = _faceDepth.ptr(); ushort* dest = _smoothedFaceDepth.ptr(); + ushort minimumDepth = numeric_limits::max(); for (int i = 0; i < ENCODED_FACE_HEIGHT; i++) { for (int j = 0; j < ENCODED_FACE_WIDTH; j++) { ushort depth = *src++; if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { - depthTotal += depth; - depthMinimum = min(depthMinimum, depth); - depthSamples++; - + minimumDepth = min(minimumDepth, depth); *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); } dest++; } } - const ushort DEPTH_MINIMUM_OFFSET = 64; - float mean = (depthSamples == 0) ? UNINITIALIZED_FACE_DEPTH : depthMinimum + DEPTH_MINIMUM_OFFSET; + const ushort MINIMUM_DEPTH_OFFSET = 64; + float midFaceDepth = minimumDepth + MINIMUM_DEPTH_OFFSET; - // smooth the mean over time - const float DEPTH_OFFSET_SMOOTHING = 0.5f; - _smoothedMeanFaceDepth = (_smoothedMeanFaceDepth == UNINITIALIZED_FACE_DEPTH) ? mean : - glm::mix(mean, _smoothedMeanFaceDepth, DEPTH_OFFSET_SMOOTHING); + // smooth the mid face depth over time + const float MID_FACE_DEPTH_SMOOTHING = 0.5f; + _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : + glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); // convert from 11 to 8 bits for preview/local display const uchar EIGHT_BIT_MIDPOINT = 128; - double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMeanFaceDepth; + double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); // likewise for the encoded representation @@ -707,7 +702,7 @@ void FrameGrabber::grabFrame() { Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", - Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMeanFaceDepth), + Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); } diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index 077e9eed98..021ef3bc15 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -62,7 +62,7 @@ public: public slots: void setEnabled(bool enabled); - void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float meanFaceDepth, + void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth, const cv::RotatedRect& faceRect, const JointVector& joints); private: @@ -118,7 +118,7 @@ private: cv::Mat _backProject; cv::Rect _searchWindow; cv::Mat _grayDepthFrame; - float _smoothedMeanFaceDepth; + float _smoothedMidFaceDepth; vpx_codec_ctx_t _colorCodec; vpx_codec_ctx_t _depthCodec; From d18a9dc49905eeffa899c370a548684e15bb3bba Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 31 Jul 2013 17:06:09 -0700 Subject: [PATCH 03/14] Working on full frame video support. --- interface/src/Webcam.cpp | 405 +++++++++++++++++++++------------------ interface/src/Webcam.h | 4 + 2 files changed, 224 insertions(+), 185 deletions(-) diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index 3457064733..b11f134fc5 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -270,7 +270,7 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame())); } -FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), +FrameGrabber::FrameGrabber() : _initialized(false), _videoSendMode(FULL_FRAME_VIDEO), _capture(0), _searchWindow(0, 0, 0, 0), _smoothedMidFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) { } @@ -364,6 +364,11 @@ static void XN_CALLBACK_TYPE calibrationCompleted(SkeletonCapability& capability } #endif +void FrameGrabber::cycleVideoSendMode() { + _videoSendMode = (VideoSendMode)((_videoSendMode + 1) % VIDEO_SEND_MODE_COUNT); + _searchWindow = cv::Rect(0, 0, 0, 0); +} + void FrameGrabber::reset() { _searchWindow = cv::Rect(0, 0, 0, 0); @@ -462,56 +467,54 @@ void FrameGrabber::grabFrame() { color = image; } - // if we don't have a search window (yet), try using the face cascade - int channels = 0; - float ranges[] = { 0, 180 }; - const float* range = ranges; - if (_searchWindow.area() == 0) { - vector faces; - _faceCascade.detectMultiScale(color, faces, 1.1, 6); - if (!faces.empty()) { - _searchWindow = faces.front(); - updateHSVFrame(color, format); - - Mat faceHsv(_hsvFrame, _searchWindow); - Mat faceMask(_mask, _searchWindow); - int sizes = 30; - calcHist(&faceHsv, 1, &channels, faceMask, _histogram, 1, &sizes, &range); - double min, max; - minMaxLoc(_histogram, &min, &max); - _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max); - } - } RotatedRect faceRect; - if (_searchWindow.area() > 0) { - updateHSVFrame(color, format); + int encodedWidth; + int encodedHeight; + int depthBitrateMultiplier = 1; + if (_videoSendMode == FULL_FRAME_VIDEO) { + // no need to find the face if we're sending full frame video + faceRect.center = Point2f(color.cols / 2.0f, color.rows / 2.0f); + faceRect.size = Size2f(color.cols, color.rows); + encodedWidth = color.cols; + encodedHeight = color.rows; - calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range); - bitwise_and(_backProject, _mask, _backProject); - - faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1)); - Rect faceBounds = faceRect.boundingRect(); - Rect imageBounds(0, 0, color.cols, color.rows); - _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); - } - - const int ENCODED_FACE_WIDTH = 128; - const int ENCODED_FACE_HEIGHT = 128; - if (_colorCodec.name == 0) { - // initialize encoder context(s) - vpx_codec_enc_cfg_t codecConfig; - vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0); - codecConfig.rc_target_bitrate = ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * - codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h; - codecConfig.g_w = ENCODED_FACE_WIDTH; - codecConfig.g_h = ENCODED_FACE_HEIGHT; - vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0); - - if (!depth.empty()) { - int DEPTH_BITRATE_MULTIPLIER = 2; - codecConfig.rc_target_bitrate *= 2; - vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0); + } else { + // if we don't have a search window (yet), try using the face cascade + int channels = 0; + float ranges[] = { 0, 180 }; + const float* range = ranges; + if (_searchWindow.area() == 0) { + vector faces; + _faceCascade.detectMultiScale(color, faces, 1.1, 6); + if (!faces.empty()) { + _searchWindow = faces.front(); + updateHSVFrame(color, format); + + Mat faceHsv(_hsvFrame, _searchWindow); + Mat faceMask(_mask, _searchWindow); + int sizes = 30; + calcHist(&faceHsv, 1, &channels, faceMask, _histogram, 1, &sizes, &range); + double min, max; + minMaxLoc(_histogram, &min, &max); + _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max); + } } + if (_searchWindow.area() > 0) { + updateHSVFrame(color, format); + + calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range); + bitwise_and(_backProject, _mask, _backProject); + + faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1)); + Rect faceBounds = faceRect.boundingRect(); + Rect imageBounds(0, 0, color.cols, color.rows); + _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); + } + const int ENCODED_FACE_WIDTH = 128; + const int ENCODED_FACE_HEIGHT = 128; + encodedWidth = ENCODED_FACE_WIDTH; + encodedHeight = ENCODED_FACE_HEIGHT; + depthBitrateMultiplier = 2; } // correct for 180 degree rotations @@ -535,150 +538,86 @@ void FrameGrabber::grabFrame() { _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); } - // resize/rotate face into encoding rectangle - _faceColor.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_8UC3); - Point2f sourcePoints[4]; - _smoothedFaceRect.points(sourcePoints); - Point2f destPoints[] = { Point2f(0, ENCODED_FACE_HEIGHT), Point2f(0, 0), Point2f(ENCODED_FACE_WIDTH, 0) }; - Mat transform = getAffineTransform(sourcePoints, destPoints); - warpAffine(color, _faceColor, transform, _faceColor.size()); - - // convert from RGB to YV12 - const int ENCODED_BITS_PER_Y = 8; - const int ENCODED_BITS_PER_VU = 2; - const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU; - const int BITS_PER_BYTE = 8; - _encodedFace.resize(ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE); - vpx_image_t vpxImage; - vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, 1, (unsigned char*)_encodedFace.data()); - uchar* yline = vpxImage.planes[0]; - uchar* vline = vpxImage.planes[1]; - uchar* uline = vpxImage.planes[2]; - const int Y_RED_WEIGHT = (int)(0.299 * 256); - const int Y_GREEN_WEIGHT = (int)(0.587 * 256); - const int Y_BLUE_WEIGHT = (int)(0.114 * 256); - const int V_RED_WEIGHT = (int)(0.713 * 256); - const int U_BLUE_WEIGHT = (int)(0.564 * 256); - int redIndex = 0; - int greenIndex = 1; - int blueIndex = 2; - if (format == GL_BGR) { - redIndex = 2; - blueIndex = 0; - } - for (int i = 0; i < ENCODED_FACE_HEIGHT; i += 2) { - uchar* ydest = yline; - uchar* vdest = vline; - uchar* udest = uline; - for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { - uchar* tl = _faceColor.ptr(i, j); - uchar* tr = _faceColor.ptr(i, j + 1); - uchar* bl = _faceColor.ptr(i + 1, j); - uchar* br = _faceColor.ptr(i + 1, j + 1); + if (_videoSendMode != NO_VIDEO) { + if (_colorCodec.name == 0) { + // initialize encoder context(s) + vpx_codec_enc_cfg_t codecConfig; + vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0); + codecConfig.rc_target_bitrate = encodedWidth * encodedHeight * + codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h; + codecConfig.g_w = encodedWidth; + codecConfig.g_h = encodedHeight; + vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0); - ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] * - Y_GREEN_WEIGHT + bl[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] * - Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest += 2; - - int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex]; - int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex]; - int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex]; - int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8; - - *vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128; - *udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128; - } - yline += vpxImage.stride[0] * 2; - vline += vpxImage.stride[1]; - uline += vpxImage.stride[2]; - } - - // encode the frame - vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); - - // start the payload off with the aspect ratio - QByteArray payload(sizeof(float), 0); - *(float*)payload.data() = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; - - // extract the encoded frame - vpx_codec_iter_t iterator = 0; - const vpx_codec_cx_pkt_t* packet; - while ((packet = vpx_codec_get_cx_data(&_colorCodec, &iterator)) != 0) { - if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { - // prepend the length, which will indicate whether there's a depth frame too - payload.append((const char*)&packet->data.frame.sz, sizeof(packet->data.frame.sz)); - payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); - } - } - - if (!depth.empty()) { - // warp the face depth without interpolation (because it will contain invalid zero values) - _faceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); - warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST); - - _smoothedFaceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); - - // smooth the depth over time - const ushort ELEVEN_BIT_MINIMUM = 0; - const ushort ELEVEN_BIT_MAXIMUM = 2047; - const float DEPTH_SMOOTHING = 0.25f; - ushort* src = _faceDepth.ptr(); - ushort* dest = _smoothedFaceDepth.ptr(); - ushort minimumDepth = numeric_limits::max(); - for (int i = 0; i < ENCODED_FACE_HEIGHT; i++) { - for (int j = 0; j < ENCODED_FACE_WIDTH; j++) { - ushort depth = *src++; - if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { - minimumDepth = min(minimumDepth, depth); - *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); - } - dest++; + if (!depth.empty()) { + codecConfig.rc_target_bitrate *= depthBitrateMultiplier; + vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0); } } - const ushort MINIMUM_DEPTH_OFFSET = 64; - float midFaceDepth = minimumDepth + MINIMUM_DEPTH_OFFSET; + + Mat transform; + if (_videoSendMode == FACE_VIDEO) { + // resize/rotate face into encoding rectangle + _faceColor.create(encodedHeight, encodedWidth, CV_8UC3); + Point2f sourcePoints[4]; + _smoothedFaceRect.points(sourcePoints); + Point2f destPoints[] = { Point2f(0, encodedHeight), Point2f(0, 0), Point2f(encodedWidth, 0) }; + transform = getAffineTransform(sourcePoints, destPoints); + warpAffine(color, _faceColor, transform, _faceColor.size()); - // smooth the mid face depth over time - const float MID_FACE_DEPTH_SMOOTHING = 0.5f; - _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : - glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); - - // convert from 11 to 8 bits for preview/local display - const uchar EIGHT_BIT_MIDPOINT = 128; - double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; - depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); - - // likewise for the encoded representation + } else { + _faceColor = color; + } + + // convert from RGB to YV12 + const int ENCODED_BITS_PER_Y = 8; + const int ENCODED_BITS_PER_VU = 2; + const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU; + const int BITS_PER_BYTE = 8; + _encodedFace.resize(encodedWidth * encodedHeight * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE); + vpx_image_t vpxImage; + vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, encodedWidth, encodedHeight, 1, + (unsigned char*)_encodedFace.data()); uchar* yline = vpxImage.planes[0]; uchar* vline = vpxImage.planes[1]; uchar* uline = vpxImage.planes[2]; - const uchar EIGHT_BIT_MAXIMUM = 255; - for (int i = 0; i < ENCODED_FACE_HEIGHT; i += 2) { + const int Y_RED_WEIGHT = (int)(0.299 * 256); + const int Y_GREEN_WEIGHT = (int)(0.587 * 256); + const int Y_BLUE_WEIGHT = (int)(0.114 * 256); + const int V_RED_WEIGHT = (int)(0.713 * 256); + const int U_BLUE_WEIGHT = (int)(0.564 * 256); + int redIndex = 0; + int greenIndex = 1; + int blueIndex = 2; + if (format == GL_BGR) { + redIndex = 2; + blueIndex = 0; + } + for (int i = 0; i < encodedHeight; i += 2) { uchar* ydest = yline; uchar* vdest = vline; uchar* udest = uline; - for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { - ushort tl = *_smoothedFaceDepth.ptr(i, j); - ushort tr = *_smoothedFaceDepth.ptr(i, j + 1); - ushort bl = *_smoothedFaceDepth.ptr(i + 1, j); - ushort br = *_smoothedFaceDepth.ptr(i + 1, j + 1); - - uchar mask = EIGHT_BIT_MAXIMUM; + for (int j = 0; j < encodedWidth; j += 2) { + uchar* tl = _faceColor.ptr(i, j); + uchar* tr = _faceColor.ptr(i, j + 1); + uchar* bl = _faceColor.ptr(i + 1, j); + uchar* br = _faceColor.ptr(i + 1, j + 1); - ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(tl + depthOffset); - ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(tr + depthOffset); - ydest[vpxImage.stride[0]] = (bl == ELEVEN_BIT_MINIMUM) ? - (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(bl + depthOffset); - ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? - (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(br + depthOffset); + ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] * + Y_GREEN_WEIGHT + bl[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] * + Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8; ydest += 2; - - *vdest++ = mask; - *udest++ = EIGHT_BIT_MIDPOINT; + + int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex]; + int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex]; + int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex]; + int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8; + + *vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128; + *udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128; } yline += vpxImage.stride[0] * 2; vline += vpxImage.stride[1]; @@ -686,21 +625,117 @@ void FrameGrabber::grabFrame() { } // encode the frame - vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME); + vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); + + // start the payload off with the aspect ratio + QByteArray payload(sizeof(float), 0); + *(float*)payload.data() = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; // extract the encoded frame vpx_codec_iter_t iterator = 0; const vpx_codec_cx_pkt_t* packet; - while ((packet = vpx_codec_get_cx_data(&_depthCodec, &iterator)) != 0) { + while ((packet = vpx_codec_get_cx_data(&_colorCodec, &iterator)) != 0) { if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { + // prepend the length, which will indicate whether there's a depth frame too + payload.append((const char*)&packet->data.frame.sz, sizeof(packet->data.frame.sz)); payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); } } + + if (!depth.empty()) { + if (_videoSendMode == FACE_VIDEO) { + // warp the face depth without interpolation (because it will contain invalid zero values) + _faceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST); + + } else { + _faceDepth = depth; + } + _smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + + // smooth the depth over time + const ushort ELEVEN_BIT_MINIMUM = 0; + const ushort ELEVEN_BIT_MAXIMUM = 2047; + const float DEPTH_SMOOTHING = 0.25f; + ushort* src = _faceDepth.ptr(); + ushort* dest = _smoothedFaceDepth.ptr(); + ushort minimumDepth = numeric_limits::max(); + for (int i = 0; i < encodedHeight; i++) { + for (int j = 0; j < encodedWidth; j++) { + ushort depth = *src++; + if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { + minimumDepth = min(minimumDepth, depth); + *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); + } + dest++; + } + } + const ushort MINIMUM_DEPTH_OFFSET = 64; + const float FIXED_MID_DEPTH = 640.0f; + float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH; + + // smooth the mid face depth over time + const float MID_FACE_DEPTH_SMOOTHING = 0.5f; + _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : + glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); + + // convert from 11 to 8 bits for preview/local display + const uchar EIGHT_BIT_MIDPOINT = 128; + double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; + depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); + + // likewise for the encoded representation + uchar* yline = vpxImage.planes[0]; + uchar* vline = vpxImage.planes[1]; + uchar* uline = vpxImage.planes[2]; + const uchar EIGHT_BIT_MAXIMUM = 255; + for (int i = 0; i < encodedHeight; i += 2) { + uchar* ydest = yline; + uchar* vdest = vline; + uchar* udest = uline; + for (int j = 0; j < encodedWidth; j += 2) { + ushort tl = *_smoothedFaceDepth.ptr(i, j); + ushort tr = *_smoothedFaceDepth.ptr(i, j + 1); + ushort bl = *_smoothedFaceDepth.ptr(i + 1, j); + ushort br = *_smoothedFaceDepth.ptr(i + 1, j + 1); + + uchar mask = EIGHT_BIT_MAXIMUM; + + ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(tl + depthOffset); + ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(tr + depthOffset); + ydest[vpxImage.stride[0]] = (bl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(bl + depthOffset); + ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(br + depthOffset); + ydest += 2; + + *vdest++ = mask; + *udest++ = EIGHT_BIT_MIDPOINT; + } + yline += vpxImage.stride[0] * 2; + vline += vpxImage.stride[1]; + uline += vpxImage.stride[2]; + } + + // encode the frame + vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME); + + // extract the encoded frame + vpx_codec_iter_t iterator = 0; + const vpx_codec_cx_pkt_t* packet; + while ((packet = vpx_codec_get_cx_data(&_depthCodec, &iterator)) != 0) { + if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { + payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); + } + } + } + + QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", + Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); } - QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", - Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); - QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index 021ef3bc15..b5a26d9b13 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -100,16 +100,20 @@ public: public slots: + void cycleVideoSendMode(); void reset(); void shutdown(); void grabFrame(); private: + enum VideoSendMode { NO_VIDEO, FACE_VIDEO, FULL_FRAME_VIDEO, VIDEO_SEND_MODE_COUNT }; + bool init(); void updateHSVFrame(const cv::Mat& frame, int format); bool _initialized; + VideoSendMode _videoSendMode; CvCapture* _capture; cv::CascadeClassifier _faceCascade; cv::Mat _hsvFrame; From a86ceb275961c259e3ac8f1ff03b3da059d98f62 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 31 Jul 2013 18:18:37 -0700 Subject: [PATCH 04/14] More work on full frame video. --- interface/src/Webcam.cpp | 8 +++++--- interface/src/avatar/Face.cpp | 27 +++++++++++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index b11f134fc5..3b1f1ec9ee 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -569,7 +569,8 @@ void FrameGrabber::grabFrame() { _faceColor = color; } - // convert from RGB to YV12 + // convert from RGB to YV12: see http://www.fourcc.org/yuv.php and + // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor const int ENCODED_BITS_PER_Y = 8; const int ENCODED_BITS_PER_VU = 2; const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU; @@ -627,9 +628,10 @@ void FrameGrabber::grabFrame() { // encode the frame vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); - // start the payload off with the aspect ratio + // start the payload off with the aspect ratio (zero for no face) QByteArray payload(sizeof(float), 0); - *(float*)payload.data() = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; + *(float*)payload.data() = (_videoSendMode == FACE_VIDEO) ? + (_smoothedFaceRect.size.width / _smoothedFaceRect.size.height) : 0.0f; // extract the encoded frame vpx_codec_iter_t iterator = 0; diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp index ff31241c54..72d1b5d5c2 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/Face.cpp @@ -97,7 +97,8 @@ int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { vpx_codec_iter_t iterator = 0; vpx_image_t* image; while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { - // convert from YV12 to RGB + // convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and + // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor Mat color(image->d_h, image->d_w, CV_8UC3); uchar* yline = image->planes[0]; uchar* vline = image->planes[1]; @@ -208,9 +209,19 @@ bool Face::render(float alpha) { glm::quat orientation = _owningHead->getOrientation(); glm::vec3 axis = glm::axis(orientation); glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); - float scale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + + float aspect, scale; + if (_aspectRatio == 0.0f) { + aspect = _textureSize.width / _textureSize.height; + const float FULL_FRAME_SCALE = 0.5f; + scale = FULL_FRAME_SCALE * _owningHead->getScale(); + + } else { + aspect = _aspectRatio; + scale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + } glScalef(scale, scale, scale); - + glColor4f(1.0f, 1.0f, 1.0f, alpha); Point2f points[4]; @@ -292,7 +303,7 @@ bool Face::render(float alpha) { (points[3].x - points[0].x) / _textureSize.width, (points[3].y - points[0].y) / _textureSize.height); _program->setUniformValue(_texCoordUpLocation, (points[1].x - points[0].x) / _textureSize.width, (points[1].y - points[0].y) / _textureSize.height); - _program->setUniformValue(_aspectRatioLocation, _aspectRatio); + _program->setUniformValue(_aspectRatioLocation, aspect); glEnableClientState(GL_VERTEX_ARRAY); glVertexPointer(2, GL_FLOAT, 0, 0); @@ -324,13 +335,13 @@ bool Face::render(float alpha) { glBegin(GL_QUADS); glTexCoord2f(points[0].x / _textureSize.width, points[0].y / _textureSize.height); - glVertex3f(0.5f, -0.5f / _aspectRatio, -0.5f); + glVertex3f(0.5f, -0.5f / aspect, -0.5f); glTexCoord2f(points[1].x / _textureSize.width, points[1].y / _textureSize.height); - glVertex3f(0.5f, 0.5f / _aspectRatio, -0.5f); + glVertex3f(0.5f, 0.5f / aspect, -0.5f); glTexCoord2f(points[2].x / _textureSize.width, points[2].y / _textureSize.height); - glVertex3f(-0.5f, 0.5f / _aspectRatio, -0.5f); + glVertex3f(-0.5f, 0.5f / aspect, -0.5f); glTexCoord2f(points[3].x / _textureSize.width, points[3].y / _textureSize.height); - glVertex3f(-0.5f, -0.5f / _aspectRatio, -0.5f); + glVertex3f(-0.5f, -0.5f / aspect, -0.5f); glEnd(); glDisable(GL_TEXTURE_2D); From 2fde29e04a1a6e7c033ba0b73065589265e68466 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 1 Aug 2013 11:53:18 -0700 Subject: [PATCH 05/14] use a 0 public address to designate server local to DS --- domain-server/src/main.cpp | 5 +++-- libraries/shared/src/NodeList.cpp | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/domain-server/src/main.cpp b/domain-server/src/main.cpp index 5d41efdc67..f62ce10e92 100644 --- a/domain-server/src/main.cpp +++ b/domain-server/src/main.cpp @@ -104,9 +104,10 @@ int main(int argc, const char * argv[]) // so hardcode the EC2 public address for now if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress) { // If we're not running "local" then we do replace the IP - // with the EC2 IP. Otherwise, we use our normal public IP + // with 0. This designates to clients that the server is reachable + // at the same IP address if (!isLocalMode) { - nodePublicAddress.sin_addr.s_addr = 895283510; // local IP in this format... + nodePublicAddress.sin_addr.s_addr = 0; destinationSocket = (sockaddr*) &nodeLocalAddress; } } diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index 42aa024e9d..8819677bcc 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -352,6 +352,12 @@ int NodeList::processDomainServerList(unsigned char* packetData, size_t dataByte readPtr += unpackSocket(readPtr, (sockaddr*) &nodePublicSocket); readPtr += unpackSocket(readPtr, (sockaddr*) &nodeLocalSocket); + // if the public socket address is 0 then it's reachable at the same IP + // as the domain server + if (nodePublicSocket.sin_addr.s_addr == 0) { + inet_aton(_domainIP, &nodePublicSocket.sin_addr); + } + addOrUpdateNode((sockaddr*) &nodePublicSocket, (sockaddr*) &nodeLocalSocket, nodeType, nodeId); } From 2b31a3586e059ee9736e1b484d37d6c416ec9e19 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 1 Aug 2013 12:01:48 -0700 Subject: [PATCH 06/14] comment changes for local vs. non-local --- domain-server/src/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/domain-server/src/main.cpp b/domain-server/src/main.cpp index f62ce10e92..13865cbacc 100644 --- a/domain-server/src/main.cpp +++ b/domain-server/src/main.cpp @@ -55,10 +55,10 @@ int main(int argc, const char * argv[]) // domain server bool isLocalMode = cmdOptionExists(argc, argv, "--local"); if (isLocalMode) { - printf("NOTE: Running in Local Mode!\n"); + printf("NOTE: Running in local mode!\n"); } else { printf("--------------------------------------------------\n"); - printf("NOTE: Running in EC2 Mode. \n"); + printf("NOTE: Not running in local mode. \n"); printf("If you're a developer testing a local system, you\n"); printf("probably want to include --local on command line.\n"); printf("--------------------------------------------------\n"); From ba9b5888d25543267ee050f6d310a4ad7f550ab5 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 1 Aug 2013 13:00:01 -0700 Subject: [PATCH 07/14] More work on full frame sending. --- interface/resources/shaders/face.vert | 6 +---- interface/src/Webcam.cpp | 17 +++++++----- interface/src/Webcam.h | 5 +++- interface/src/avatar/Avatar.cpp | 19 +++++++------ interface/src/avatar/Face.cpp | 39 ++++++++++++++++----------- interface/src/avatar/Face.h | 11 ++++---- 6 files changed, 57 insertions(+), 40 deletions(-) diff --git a/interface/resources/shaders/face.vert b/interface/resources/shaders/face.vert index 358a8902fe..018a85f524 100644 --- a/interface/resources/shaders/face.vert +++ b/interface/resources/shaders/face.vert @@ -17,9 +17,6 @@ uniform vec2 texCoordRight; // the texture coordinate vector from bottom to the top uniform vec2 texCoordUp; -// the aspect ratio of the image -uniform float aspectRatio; - // the depth texture uniform sampler2D depthTexture; @@ -31,6 +28,5 @@ void main(void) { const float MIN_VISIBLE_DEPTH = 1.0 / 255.0; const float MAX_VISIBLE_DEPTH = 254.0 / 255.0; gl_FrontColor = vec4(1.0, 1.0, 1.0, step(MIN_VISIBLE_DEPTH, depth) * (1.0 - step(MAX_VISIBLE_DEPTH, depth))); - gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, - (gl_Vertex.y - 0.5) / aspectRatio, depth * 2.0 - 2.0, 1.0); + gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, gl_Vertex.y - 0.5, depth - 0.5, 1.0); } diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index 3b1f1ec9ee..8340e8ba06 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -19,6 +19,7 @@ #include "Application.h" #include "Webcam.h" +#include "avatar/Face.h" using namespace cv; using namespace std; @@ -155,7 +156,7 @@ Webcam::~Webcam() { const float METERS_PER_MM = 1.0f / 1000.0f; void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midFaceDepth, - const RotatedRect& faceRect, const JointVector& joints) { + float aspectRatio, const RotatedRect& faceRect, const JointVector& joints) { IplImage colorImage = color; glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); if (_colorTextureID == 0) { @@ -192,7 +193,8 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); glBindTexture(GL_TEXTURE_2D, 0); - // store our face rect and joints, update our frame count for fps computation + // store our various data, update our frame count for fps computation + _aspectRatio = aspectRatio; _faceRect = faceRect; _joints = joints; _frameCount++; @@ -538,6 +540,10 @@ void FrameGrabber::grabFrame() { _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); } + // the aspect ratio is derived from the face rect dimensions unless we're full-frame + float aspectRatio = (_videoSendMode == FULL_FRAME_VIDEO) ? FULL_FRAME_ASPECT : + (_smoothedFaceRect.size.width / _smoothedFaceRect.size.height); + if (_videoSendMode != NO_VIDEO) { if (_colorCodec.name == 0) { // initialize encoder context(s) @@ -628,10 +634,9 @@ void FrameGrabber::grabFrame() { // encode the frame vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); - // start the payload off with the aspect ratio (zero for no face) + // start the payload off with the aspect ratio (zero for full frame) QByteArray payload(sizeof(float), 0); - *(float*)payload.data() = (_videoSendMode == FACE_VIDEO) ? - (_smoothedFaceRect.size.width / _smoothedFaceRect.size.height) : 0.0f; + *(float*)payload.data() = aspectRatio; // extract the encoded frame vpx_codec_iter_t iterator = 0; @@ -740,7 +745,7 @@ void FrameGrabber::grabFrame() { QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), - Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); + Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); } bool FrameGrabber::init() { diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index b5a26d9b13..cc5c313ece 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -50,6 +50,8 @@ public: GLuint getDepthTextureID() const { return _depthTextureID; } const cv::Size2f& getTextureSize() const { return _textureSize; } + float getAspectRatio() const { return _aspectRatio; } + const cv::RotatedRect& getFaceRect() const { return _faceRect; } const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; } @@ -63,7 +65,7 @@ public slots: void setEnabled(bool enabled); void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth, - const cv::RotatedRect& faceRect, const JointVector& joints); + float aspectRatio, const cv::RotatedRect& faceRect, const JointVector& joints); private: @@ -75,6 +77,7 @@ private: GLuint _colorTextureID; GLuint _depthTextureID; cv::Size2f _textureSize; + float _aspectRatio; cv::RotatedRect _faceRect; cv::RotatedRect _initialFaceRect; float _initialFaceDepth; diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 0bf2c94b10..bddc5d1df1 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -314,10 +314,7 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook, estimatedPosition = webcam->getEstimatedPosition(); // apply face data - _head.getFace().setColorTextureID(webcam->getColorTextureID()); - _head.getFace().setDepthTextureID(webcam->getDepthTextureID()); - _head.getFace().setTextureSize(webcam->getTextureSize()); - _head.getFace().setTextureRect(webcam->getFaceRect()); + _head.getFace().setFrameFromWebcam(); // compute and store the joint rotations const JointVector& joints = webcam->getEstimatedJoints(); @@ -334,7 +331,7 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook, } } } else { - _head.getFace().setColorTextureID(0); + _head.getFace().clearFrame(); } _head.setPitch(estimatedRotation.x * amplifyAngle.x + pitchFromTouch); _head.setYaw(estimatedRotation.y * amplifyAngle.y + yawFromTouch); @@ -1300,9 +1297,15 @@ float Avatar::getBallRenderAlpha(int ball, bool lookingInMirror) const { } void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { - - // Render the body as balls and cones - if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) { + + if (_head.getFace().isFullFrame()) { + // Render the full-frame video + float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); + if (alpha > 0.0f) { + _head.getFace().render(1.0f); + } + } else if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) { + // Render the body as balls and cones for (int b = 0; b < NUM_AVATAR_BODY_BALLS; b++) { float alpha = getBallRenderAlpha(b, lookingInMirror); diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp index 72d1b5d5c2..d982d1a4a5 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/Face.cpp @@ -17,6 +17,7 @@ #include "Avatar.h" #include "Head.h" #include "Face.h" +#include "Webcam.h" #include "renderer/ProgramObject.h" using namespace cv; @@ -25,7 +26,6 @@ ProgramObject* Face::_program = 0; int Face::_texCoordCornerLocation; int Face::_texCoordRightLocation; int Face::_texCoordUpLocation; -int Face::_aspectRatioLocation; GLuint Face::_vboID; GLuint Face::_iboID; @@ -55,11 +55,19 @@ Face::~Face() { } } -void Face::setTextureRect(const cv::RotatedRect& textureRect) { - _textureRect = textureRect; - _aspectRatio = _textureRect.size.width / _textureRect.size.height; +void Face::setFrameFromWebcam() { + Webcam* webcam = Application::getInstance()->getWebcam(); + _colorTextureID = webcam->getColorTextureID(); + _depthTextureID = webcam->getDepthTextureID(); + _textureSize = webcam->getTextureSize(); + _textureRect = webcam->getFaceRect(); + _aspectRatio = webcam->getAspectRatio(); } +void Face::clearFrame() { + _colorTextureID = 0; +} + int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { if (_colorCodec.name == 0) { // initialize decoder context @@ -210,17 +218,20 @@ bool Face::render(float alpha) { glm::vec3 axis = glm::axis(orientation); glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); - float aspect, scale; - if (_aspectRatio == 0.0f) { + float aspect, xScale, zScale; + if (_aspectRatio == FULL_FRAME_ASPECT) { aspect = _textureSize.width / _textureSize.height; const float FULL_FRAME_SCALE = 0.5f; - scale = FULL_FRAME_SCALE * _owningHead->getScale(); + xScale = FULL_FRAME_SCALE * _owningHead->getScale(); + zScale = xScale * 0.3f; } else { aspect = _aspectRatio; - scale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + xScale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + zScale = xScale * 1.5f; + glTranslatef(0.0f, -xScale * 0.75f, -xScale); } - glScalef(scale, scale, scale); + glScalef(xScale, xScale / aspect, zScale); glColor4f(1.0f, 1.0f, 1.0f, alpha); @@ -254,7 +265,6 @@ bool Face::render(float alpha) { _texCoordCornerLocation = _program->uniformLocation("texCoordCorner"); _texCoordRightLocation = _program->uniformLocation("texCoordRight"); _texCoordUpLocation = _program->uniformLocation("texCoordUp"); - _aspectRatioLocation = _program->uniformLocation("aspectRatio"); glGenBuffers(1, &_vboID); glBindBuffer(GL_ARRAY_BUFFER, _vboID); @@ -303,7 +313,6 @@ bool Face::render(float alpha) { (points[3].x - points[0].x) / _textureSize.width, (points[3].y - points[0].y) / _textureSize.height); _program->setUniformValue(_texCoordUpLocation, (points[1].x - points[0].x) / _textureSize.width, (points[1].y - points[0].y) / _textureSize.height); - _program->setUniformValue(_aspectRatioLocation, aspect); glEnableClientState(GL_VERTEX_ARRAY); glVertexPointer(2, GL_FLOAT, 0, 0); @@ -335,13 +344,13 @@ bool Face::render(float alpha) { glBegin(GL_QUADS); glTexCoord2f(points[0].x / _textureSize.width, points[0].y / _textureSize.height); - glVertex3f(0.5f, -0.5f / aspect, -0.5f); + glVertex3f(0.5f, -0.5f, 0.0f); glTexCoord2f(points[1].x / _textureSize.width, points[1].y / _textureSize.height); - glVertex3f(0.5f, 0.5f / aspect, -0.5f); + glVertex3f(0.5f, 0.5f, 0.0f); glTexCoord2f(points[2].x / _textureSize.width, points[2].y / _textureSize.height); - glVertex3f(-0.5f, 0.5f / aspect, -0.5f); + glVertex3f(-0.5f, 0.5f, 0.0f); glTexCoord2f(points[3].x / _textureSize.width, points[3].y / _textureSize.height); - glVertex3f(-0.5f, -0.5f / aspect, -0.5f); + glVertex3f(-0.5f, -0.5f, 0.0f); glEnd(); glDisable(GL_TEXTURE_2D); diff --git a/interface/src/avatar/Face.h b/interface/src/avatar/Face.h index d4812fecfb..a40111dcd3 100644 --- a/interface/src/avatar/Face.h +++ b/interface/src/avatar/Face.h @@ -20,6 +20,8 @@ class Head; class ProgramObject; +const float FULL_FRAME_ASPECT = 0.0f; + class Face : public QObject { Q_OBJECT @@ -28,10 +30,10 @@ public: Face(Head* owningHead); ~Face(); - void setColorTextureID(GLuint colorTextureID) { _colorTextureID = colorTextureID; } - void setDepthTextureID(GLuint depthTextureID) { _depthTextureID = depthTextureID; } - void setTextureSize(const cv::Size2f& textureSize) { _textureSize = textureSize; } - void setTextureRect(const cv::RotatedRect& textureRect); + bool isFullFrame() const { return _colorTextureID != 0 && _aspectRatio == FULL_FRAME_ASPECT; } + + void setFrameFromWebcam(); + void clearFrame(); int processVideoMessage(unsigned char* packetData, size_t dataBytes); @@ -68,7 +70,6 @@ private: static int _texCoordCornerLocation; static int _texCoordRightLocation; static int _texCoordUpLocation; - static int _aspectRatioLocation; static GLuint _vboID; static GLuint _iboID; }; From 9ba0e3bb0c25fb745ebf42d5a6d7c11772b23358 Mon Sep 17 00:00:00 2001 From: Mark Peng Date: Thu, 1 Aug 2013 13:03:17 -0700 Subject: [PATCH 08/14] Add reference lines to mouse voxel when adding/deleting voxels. --- interface/src/Application.cpp | 3 ++- interface/src/Util.cpp | 26 ++++++++++++++++++++++++++ interface/src/Util.h | 3 ++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 9b292b3d50..7fc9bd7ac8 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2676,13 +2676,14 @@ void Application::displaySide(Camera& whichCamera) { if (_mouseVoxel.s != 0) { glDisable(GL_LIGHTING); glPushMatrix(); + glScalef(TREE_SCALE, TREE_SCALE, TREE_SCALE); + renderMouseVoxelGrid(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); if (_addVoxelMode->isChecked()) { // use a contrasting color so that we can see what we're doing glColor3ub(_mouseVoxel.red + 128, _mouseVoxel.green + 128, _mouseVoxel.blue + 128); } else { glColor3ub(_mouseVoxel.red, _mouseVoxel.green, _mouseVoxel.blue); } - glScalef(TREE_SCALE, TREE_SCALE, TREE_SCALE); glTranslatef(_mouseVoxel.x + _mouseVoxel.s*0.5f, _mouseVoxel.y + _mouseVoxel.s*0.5f, _mouseVoxel.z + _mouseVoxel.s*0.5f); diff --git a/interface/src/Util.cpp b/interface/src/Util.cpp index 0f2c3a8955..d2f9e56462 100644 --- a/interface/src/Util.cpp +++ b/interface/src/Util.cpp @@ -365,7 +365,33 @@ void renderGroundPlaneGrid(float size, float impact) { glEnd(); } +void renderMouseVoxelGrid(const float& mouseVoxelX, const float& mouseVoxelY, const float& mouseVoxelZ, const float& mouseVoxelS) { + glm::vec3 origin = glm::vec3(mouseVoxelX, mouseVoxelY, mouseVoxelZ); + glLineWidth(3.0); + + const int HALF_GRID_DIMENSIONS = 4; + glBegin(GL_LINES); + + glm::vec3 xColor(0.0, 0.6, 0.0); + glColor3fv(&xColor.x); + + glVertex3f(origin.x + HALF_GRID_DIMENSIONS * mouseVoxelS, 0, origin.z); + glVertex3f(origin.x - HALF_GRID_DIMENSIONS * mouseVoxelS, 0, origin.z); + + glm::vec3 zColor(0.0, 0.0, 0.6); + glColor3fv(&zColor.x); + + glVertex3f(origin.x, 0, origin.z + HALF_GRID_DIMENSIONS * mouseVoxelS); + glVertex3f(origin.x, 0, origin.z - HALF_GRID_DIMENSIONS * mouseVoxelS); + + glm::vec3 yColor(0.6, 0.0, 0.0); + glColor3fv(&yColor.x); + + glVertex3f(origin.x, 0, origin.z); + glVertex3f(origin.x, origin.y, origin.z); + glEnd(); +} void renderDiskShadow(glm::vec3 position, glm::vec3 upDirection, float radius, float darkness) { diff --git a/interface/src/Util.h b/interface/src/Util.h index fe59637a42..67ffebf4b3 100644 --- a/interface/src/Util.h +++ b/interface/src/Util.h @@ -59,8 +59,9 @@ double diffclock(timeval *clock1,timeval *clock2); void renderGroundPlaneGrid(float size, float impact); -void renderCollisionOverlay(int width, int height, float magnitude); +void renderMouseVoxelGrid(const float& mouseVoxelX, const float& mouseVoxelY, const float& mouseVoxelZ, const float& mouseVoxelS); +void renderCollisionOverlay(int width, int height, float magnitude); void renderDiskShadow(glm::vec3 position, glm::vec3 upDirection, float radius, float darkness); From 269e040263e2c7026fd473aa12faeeca10445d63 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 1 Aug 2013 15:27:55 -0700 Subject: [PATCH 09/14] Qt5 find package cleanup --- libraries/avatars/CMakeLists.txt | 4 ---- libraries/shared/CMakeLists.txt | 4 ---- libraries/voxels/CMakeLists.txt | 4 ++++ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/libraries/avatars/CMakeLists.txt b/libraries/avatars/CMakeLists.txt index 7968b5a003..207057e244 100644 --- a/libraries/avatars/CMakeLists.txt +++ b/libraries/avatars/CMakeLists.txt @@ -8,13 +8,9 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cm set(TARGET_NAME avatars) -find_package(Qt5Core) - include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) -qt5_use_modules(${TARGET_NAME} Core) - include(${MACRO_DIR}/IncludeGLM.cmake) include_glm(${TARGET_NAME} ${ROOT_DIR}) diff --git a/libraries/shared/CMakeLists.txt b/libraries/shared/CMakeLists.txt index b0c2771f48..cf1c603b7d 100644 --- a/libraries/shared/CMakeLists.txt +++ b/libraries/shared/CMakeLists.txt @@ -6,13 +6,9 @@ set(MACRO_DIR ${ROOT_DIR}/cmake/macros) set(TARGET_NAME shared) project(${TARGET_NAME}) -find_package(Qt5Core REQUIRED) - include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) -qt5_use_modules(${TARGET_NAME} Core) - set(EXTERNAL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external) if (WIN32) diff --git a/libraries/voxels/CMakeLists.txt b/libraries/voxels/CMakeLists.txt index 861001ed35..b6f7ae9718 100644 --- a/libraries/voxels/CMakeLists.txt +++ b/libraries/voxels/CMakeLists.txt @@ -8,9 +8,13 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cm set(TARGET_NAME voxels) +find_package(Qt5Widgets REQUIRED) + include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) +qt5_use_modules(${TARGET_NAME} Widgets) + include(${MACRO_DIR}/IncludeGLM.cmake) include_glm(${TARGET_NAME} ${ROOT_DIR}) From 12f42358d58df85b63ab54ac5a22f850c093e870 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 1 Aug 2013 15:39:53 -0700 Subject: [PATCH 10/14] Allow cycling between video send modes. --- interface/src/Application.cpp | 8 +- interface/src/Webcam.cpp | 192 ++++++++++++----------- interface/src/Webcam.h | 7 +- interface/src/avatar/Face.cpp | 285 ++++++++++++++++++++-------------- interface/src/avatar/Face.h | 3 + 5 files changed, 284 insertions(+), 211 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f331ee9407..ad40edd0db 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -960,14 +960,15 @@ void Application::sendAvatarFaceVideoMessage(int frameCount, const QByteArray& d int headerSize = packetPosition - packet; - // break the data up into submessages of the maximum size + // break the data up into submessages of the maximum size (at least one, for zero-length packets) *offsetPosition = 0; - while (*offsetPosition < data.size()) { + do { int payloadSize = min(data.size() - (int)*offsetPosition, MAX_PACKET_SIZE - headerSize); memcpy(packetPosition, data.constData() + *offsetPosition, payloadSize); getInstance()->controlledBroadcastToNodes(packet, headerSize + payloadSize, &NODE_TYPE_AVATAR_MIXER, 1); *offsetPosition += payloadSize; - } + + } while (*offsetPosition < data.size()); } // Every second, check the frame rates and other stuff @@ -1723,6 +1724,7 @@ void Application::initMenu() { _testPing->setChecked(true); (_fullScreenMode = optionsMenu->addAction("Fullscreen", this, SLOT(setFullscreen(bool)), Qt::Key_F))->setCheckable(true); optionsMenu->addAction("Webcam", &_webcam, SLOT(setEnabled(bool)))->setCheckable(true); + optionsMenu->addAction("Cycle Webcam Send Mode", _webcam.getGrabber(), SLOT(cycleVideoSendMode())); optionsMenu->addAction("Go Home", this, SLOT(goHome())); QMenu* renderMenu = menuBar->addMenu("Render"); diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index 8340e8ba06..b30d72aec6 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -156,7 +156,7 @@ Webcam::~Webcam() { const float METERS_PER_MM = 1.0f / 1000.0f; void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midFaceDepth, - float aspectRatio, const RotatedRect& faceRect, const JointVector& joints) { + float aspectRatio, const RotatedRect& faceRect, bool sending, const JointVector& joints) { IplImage colorImage = color; glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); if (_colorTextureID == 0) { @@ -196,6 +196,7 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF // store our various data, update our frame count for fps computation _aspectRatio = aspectRatio; _faceRect = faceRect; + _sending = sending; _joints = joints; _frameCount++; @@ -369,6 +370,8 @@ static void XN_CALLBACK_TYPE calibrationCompleted(SkeletonCapability& capability void FrameGrabber::cycleVideoSendMode() { _videoSendMode = (VideoSendMode)((_videoSendMode + 1) % VIDEO_SEND_MODE_COUNT); _searchWindow = cv::Rect(0, 0, 0, 0); + + destroyCodecs(); } void FrameGrabber::reset() { @@ -386,14 +389,7 @@ void FrameGrabber::shutdown() { cvReleaseCapture(&_capture); _capture = 0; } - if (_colorCodec.name != 0) { - vpx_codec_destroy(&_colorCodec); - _colorCodec.name = 0; - } - if (_depthCodec.name != 0) { - vpx_codec_destroy(&_depthCodec); - _depthCodec.name = 0; - } + destroyCodecs(); _initialized = false; thread()->quit(); @@ -469,16 +465,17 @@ void FrameGrabber::grabFrame() { color = image; } - RotatedRect faceRect; int encodedWidth; int encodedHeight; int depthBitrateMultiplier = 1; + Mat faceTransform; + float aspectRatio; if (_videoSendMode == FULL_FRAME_VIDEO) { // no need to find the face if we're sending full frame video - faceRect.center = Point2f(color.cols / 2.0f, color.rows / 2.0f); - faceRect.size = Size2f(color.cols, color.rows); + _smoothedFaceRect = RotatedRect(Point2f(color.cols / 2.0f, color.rows / 2.0f), Size2f(color.cols, color.rows), 0.0f); encodedWidth = color.cols; encodedHeight = color.rows; + aspectRatio = FULL_FRAME_ASPECT; } else { // if we don't have a search window (yet), try using the face cascade @@ -501,6 +498,7 @@ void FrameGrabber::grabFrame() { _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max); } } + RotatedRect faceRect; if (_searchWindow.area() > 0) { updateHSVFrame(color, format); @@ -517,33 +515,81 @@ void FrameGrabber::grabFrame() { encodedWidth = ENCODED_FACE_WIDTH; encodedHeight = ENCODED_FACE_HEIGHT; depthBitrateMultiplier = 2; + + // correct for 180 degree rotations + if (faceRect.angle < -90.0f) { + faceRect.angle += 180.0f; + + } else if (faceRect.angle > 90.0f) { + faceRect.angle -= 180.0f; + } + + // compute the smoothed face rect + if (_smoothedFaceRect.size.area() == 0) { + _smoothedFaceRect = faceRect; + + } else { + const float FACE_RECT_SMOOTHING = 0.9f; + _smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING); + _smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING); + _smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING); + _smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING); + _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); + } + + // use the face rect to compute the face transform, aspect ratio + Point2f sourcePoints[4]; + _smoothedFaceRect.points(sourcePoints); + Point2f destPoints[] = { Point2f(0, encodedHeight), Point2f(0, 0), Point2f(encodedWidth, 0) }; + faceTransform = getAffineTransform(sourcePoints, destPoints); + aspectRatio = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; + } + + const ushort ELEVEN_BIT_MINIMUM = 0; + const uchar EIGHT_BIT_MIDPOINT = 128; + double depthOffset; + if (!depth.empty()) { + if (_videoSendMode == FACE_VIDEO) { + // warp the face depth without interpolation (because it will contain invalid zero values) + _faceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + warpAffine(depth, _faceDepth, faceTransform, _faceDepth.size(), INTER_NEAREST); + + } else { + _faceDepth = depth; + } + _smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + + // smooth the depth over time + const ushort ELEVEN_BIT_MAXIMUM = 2047; + const float DEPTH_SMOOTHING = 0.25f; + ushort* src = _faceDepth.ptr(); + ushort* dest = _smoothedFaceDepth.ptr(); + ushort minimumDepth = numeric_limits::max(); + for (int i = 0; i < encodedHeight; i++) { + for (int j = 0; j < encodedWidth; j++) { + ushort depth = *src++; + if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { + minimumDepth = min(minimumDepth, depth); + *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); + } + dest++; + } + } + const ushort MINIMUM_DEPTH_OFFSET = 64; + const float FIXED_MID_DEPTH = 640.0f; + float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH; + + // smooth the mid face depth over time + const float MID_FACE_DEPTH_SMOOTHING = 0.5f; + _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : + glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); + + // convert from 11 to 8 bits for preview/local display + depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; + depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); } - // correct for 180 degree rotations - if (faceRect.angle < -90.0f) { - faceRect.angle += 180.0f; - - } else if (faceRect.angle > 90.0f) { - faceRect.angle -= 180.0f; - } - - // compute the smoothed face rect - if (_smoothedFaceRect.size.area() == 0) { - _smoothedFaceRect = faceRect; - - } else { - const float FACE_RECT_SMOOTHING = 0.9f; - _smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING); - _smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING); - _smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING); - _smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING); - _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); - } - - // the aspect ratio is derived from the face rect dimensions unless we're full-frame - float aspectRatio = (_videoSendMode == FULL_FRAME_VIDEO) ? FULL_FRAME_ASPECT : - (_smoothedFaceRect.size.width / _smoothedFaceRect.size.height); - + QByteArray payload; if (_videoSendMode != NO_VIDEO) { if (_colorCodec.name == 0) { // initialize encoder context(s) @@ -565,11 +611,7 @@ void FrameGrabber::grabFrame() { if (_videoSendMode == FACE_VIDEO) { // resize/rotate face into encoding rectangle _faceColor.create(encodedHeight, encodedWidth, CV_8UC3); - Point2f sourcePoints[4]; - _smoothedFaceRect.points(sourcePoints); - Point2f destPoints[] = { Point2f(0, encodedHeight), Point2f(0, 0), Point2f(encodedWidth, 0) }; - transform = getAffineTransform(sourcePoints, destPoints); - warpAffine(color, _faceColor, transform, _faceColor.size()); + warpAffine(color, _faceColor, faceTransform, _faceColor.size()); } else { _faceColor = color; @@ -635,8 +677,7 @@ void FrameGrabber::grabFrame() { vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); // start the payload off with the aspect ratio (zero for full frame) - QByteArray payload(sizeof(float), 0); - *(float*)payload.data() = aspectRatio; + payload.append((const char*)&aspectRatio, sizeof(float)); // extract the encoded frame vpx_codec_iter_t iterator = 0; @@ -650,48 +691,7 @@ void FrameGrabber::grabFrame() { } if (!depth.empty()) { - if (_videoSendMode == FACE_VIDEO) { - // warp the face depth without interpolation (because it will contain invalid zero values) - _faceDepth.create(encodedHeight, encodedWidth, CV_16UC1); - warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST); - - } else { - _faceDepth = depth; - } - _smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1); - - // smooth the depth over time - const ushort ELEVEN_BIT_MINIMUM = 0; - const ushort ELEVEN_BIT_MAXIMUM = 2047; - const float DEPTH_SMOOTHING = 0.25f; - ushort* src = _faceDepth.ptr(); - ushort* dest = _smoothedFaceDepth.ptr(); - ushort minimumDepth = numeric_limits::max(); - for (int i = 0; i < encodedHeight; i++) { - for (int j = 0; j < encodedWidth; j++) { - ushort depth = *src++; - if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { - minimumDepth = min(minimumDepth, depth); - *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); - } - dest++; - } - } - const ushort MINIMUM_DEPTH_OFFSET = 64; - const float FIXED_MID_DEPTH = 640.0f; - float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH; - - // smooth the mid face depth over time - const float MID_FACE_DEPTH_SMOOTHING = 0.5f; - _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : - glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); - - // convert from 11 to 8 bits for preview/local display - const uchar EIGHT_BIT_MIDPOINT = 128; - double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; - depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); - - // likewise for the encoded representation + // convert with mask uchar* yline = vpxImage.planes[0]; uchar* vline = vpxImage.planes[1]; uchar* uline = vpxImage.planes[2]; @@ -738,14 +738,15 @@ void FrameGrabber::grabFrame() { } } } - - QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", - Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); } + QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", + Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); + QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), - Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); + Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(bool, !payload.isEmpty()), + Q_ARG(JointVector, joints)); } bool FrameGrabber::init() { @@ -814,6 +815,17 @@ void FrameGrabber::updateHSVFrame(const Mat& frame, int format) { inRange(_hsvFrame, Scalar(0, 55, 65), Scalar(180, 256, 256), _mask); } +void FrameGrabber::destroyCodecs() { + if (_colorCodec.name != 0) { + vpx_codec_destroy(&_colorCodec); + _colorCodec.name = 0; + } + if (_depthCodec.name != 0) { + vpx_codec_destroy(&_depthCodec); + _depthCodec.name = 0; + } +} + Joint::Joint(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& projected) : isValid(true), position(position), rotation(rotation), projected(projected) { } diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index cc5c313ece..12ef2558db 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -44,7 +44,10 @@ public: Webcam(); ~Webcam(); + FrameGrabber* getGrabber() { return _grabber; } + bool isActive() const { return _active; } + bool isSending() const { return _sending; } GLuint getColorTextureID() const { return _colorTextureID; } GLuint getDepthTextureID() const { return _depthTextureID; } @@ -65,7 +68,7 @@ public slots: void setEnabled(bool enabled); void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth, - float aspectRatio, const cv::RotatedRect& faceRect, const JointVector& joints); + float aspectRatio, const cv::RotatedRect& faceRect, bool sending, const JointVector& joints); private: @@ -74,6 +77,7 @@ private: bool _enabled; bool _active; + bool _sending; GLuint _colorTextureID; GLuint _depthTextureID; cv::Size2f _textureSize; @@ -114,6 +118,7 @@ private: bool init(); void updateHSVFrame(const cv::Mat& frame, int format); + void destroyCodecs(); bool _initialized; VideoSendMode _videoSendMode; diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp index d982d1a4a5..0541b3c21c 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/Face.cpp @@ -57,11 +57,16 @@ Face::~Face() { void Face::setFrameFromWebcam() { Webcam* webcam = Application::getInstance()->getWebcam(); - _colorTextureID = webcam->getColorTextureID(); - _depthTextureID = webcam->getDepthTextureID(); - _textureSize = webcam->getTextureSize(); - _textureRect = webcam->getFaceRect(); - _aspectRatio = webcam->getAspectRatio(); + if (webcam->isSending()) { + _colorTextureID = webcam->getColorTextureID(); + _depthTextureID = webcam->getDepthTextureID(); + _textureSize = webcam->getTextureSize(); + _textureRect = webcam->getFaceRect(); + _aspectRatio = webcam->getAspectRatio(); + + } else { + clearFrame(); + } } void Face::clearFrame() { @@ -69,11 +74,6 @@ void Face::clearFrame() { } int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { - if (_colorCodec.name == 0) { - // initialize decoder context - vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0); - } - // skip the header unsigned char* packetPosition = packetData; int frameCount = *(uint32_t*)packetPosition; @@ -97,111 +97,135 @@ int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { int payloadSize = dataBytes - (packetPosition - packetData); memcpy(_arrivingFrame.data() + frameOffset, packetPosition, payloadSize); - if ((_frameBytesRemaining -= payloadSize) <= 0) { - float aspectRatio = *(const float*)_arrivingFrame.constData(); - size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float)); - const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t)); - vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0); - vpx_codec_iter_t iterator = 0; - vpx_image_t* image; - while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { - // convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and - // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor - Mat color(image->d_h, image->d_w, CV_8UC3); - uchar* yline = image->planes[0]; - uchar* vline = image->planes[1]; - uchar* uline = image->planes[2]; - const int RED_V_WEIGHT = (int)(1.403 * 256); - const int GREEN_V_WEIGHT = (int)(0.714 * 256); - const int GREEN_U_WEIGHT = (int)(0.344 * 256); - const int BLUE_U_WEIGHT = (int)(1.773 * 256); - for (int i = 0; i < image->d_h; i += 2) { - uchar* ysrc = yline; - uchar* vsrc = vline; - uchar* usrc = uline; - for (int j = 0; j < image->d_w; j += 2) { - uchar* tl = color.ptr(i, j); - uchar* tr = color.ptr(i, j + 1); - uchar* bl = color.ptr(i + 1, j); - uchar* br = color.ptr(i + 1, j + 1); - - int v = *vsrc++ - 128; - int u = *usrc++ - 128; - - int redOffset = (RED_V_WEIGHT * v) >> 8; - int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; - int blueOffset = (BLUE_U_WEIGHT * u) >> 8; - - int ytl = ysrc[0]; - int ytr = ysrc[1]; - int ybl = ysrc[image->w]; - int ybr = ysrc[image->w + 1]; - ysrc += 2; - - tl[0] = ytl + redOffset; - tl[1] = ytl - greenOffset; - tl[2] = ytl + blueOffset; - - tr[0] = ytr + redOffset; - tr[1] = ytr - greenOffset; - tr[2] = ytr + blueOffset; - - bl[0] = ybl + redOffset; - bl[1] = ybl - greenOffset; - bl[2] = ybl + blueOffset; - - br[0] = ybr + redOffset; - br[1] = ybr - greenOffset; - br[2] = ybr + blueOffset; - } - yline += image->stride[0] * 2; - vline += image->stride[1]; - uline += image->stride[2]; + if ((_frameBytesRemaining -= payloadSize) > 0) { + return dataBytes; // wait for the rest of the frame + } + + if (frameSize == 0) { + // destroy the codecs, if we have any + destroyCodecs(); + + // disables video data + QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, Mat()), + Q_ARG(cv::Mat, Mat()), Q_ARG(float, 0.0f)); + return dataBytes; + } + + // the switch from full frame to not (or vice versa) requires us to reinit the codecs + float aspectRatio = *(const float*)_arrivingFrame.constData(); + bool fullFrame = (aspectRatio == FULL_FRAME_ASPECT); + if (fullFrame != _lastFullFrame) { + destroyCodecs(); + _lastFullFrame = fullFrame; + } + + if (_colorCodec.name == 0) { + // initialize decoder context + vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0); + } + + size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float)); + const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t)); + vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0); + vpx_codec_iter_t iterator = 0; + vpx_image_t* image; + while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { + // convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and + // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor + Mat color(image->d_h, image->d_w, CV_8UC3); + uchar* yline = image->planes[0]; + uchar* vline = image->planes[1]; + uchar* uline = image->planes[2]; + const int RED_V_WEIGHT = (int)(1.403 * 256); + const int GREEN_V_WEIGHT = (int)(0.714 * 256); + const int GREEN_U_WEIGHT = (int)(0.344 * 256); + const int BLUE_U_WEIGHT = (int)(1.773 * 256); + for (int i = 0; i < image->d_h; i += 2) { + uchar* ysrc = yline; + uchar* vsrc = vline; + uchar* usrc = uline; + for (int j = 0; j < image->d_w; j += 2) { + uchar* tl = color.ptr(i, j); + uchar* tr = color.ptr(i, j + 1); + uchar* bl = color.ptr(i + 1, j); + uchar* br = color.ptr(i + 1, j + 1); + + int v = *vsrc++ - 128; + int u = *usrc++ - 128; + + int redOffset = (RED_V_WEIGHT * v) >> 8; + int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; + int blueOffset = (BLUE_U_WEIGHT * u) >> 8; + + int ytl = ysrc[0]; + int ytr = ysrc[1]; + int ybl = ysrc[image->w]; + int ybr = ysrc[image->w + 1]; + ysrc += 2; + + tl[0] = ytl + redOffset; + tl[1] = ytl - greenOffset; + tl[2] = ytl + blueOffset; + + tr[0] = ytr + redOffset; + tr[1] = ytr - greenOffset; + tr[2] = ytr + blueOffset; + + bl[0] = ybl + redOffset; + bl[1] = ybl - greenOffset; + bl[2] = ybl + blueOffset; + + br[0] = ybr + redOffset; + br[1] = ybr - greenOffset; + br[2] = ybr + blueOffset; } - Mat depth; - - const uint8_t* depthData = colorData + colorSize; - int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData()); - if (depthSize > 0) { - if (_depthCodec.name == 0) { - // initialize decoder context - vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0); - } - vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0); - vpx_codec_iter_t iterator = 0; - vpx_image_t* image; - while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) { - depth.create(image->d_h, image->d_w, CV_8UC1); - uchar* yline = image->planes[0]; - uchar* vline = image->planes[1]; - const uchar EIGHT_BIT_MAXIMUM = 255; - const uchar MASK_THRESHOLD = 192; - for (int i = 0; i < image->d_h; i += 2) { - uchar* ysrc = yline; - uchar* vsrc = vline; - for (int j = 0; j < image->d_w; j += 2) { - if (*vsrc++ < MASK_THRESHOLD) { - *depth.ptr(i, j) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM; - - } else { - *depth.ptr(i, j) = ysrc[0]; - *depth.ptr(i, j + 1) = ysrc[1]; - *depth.ptr(i + 1, j) = ysrc[image->stride[0]]; - *depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1]; - } - ysrc += 2; - } - yline += image->stride[0] * 2; - vline += image->stride[1]; - } - } - } - QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color), - Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio)); + yline += image->stride[0] * 2; + vline += image->stride[1]; + uline += image->stride[2]; } + Mat depth; + + const uint8_t* depthData = colorData + colorSize; + int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData()); + if (depthSize > 0) { + if (_depthCodec.name == 0) { + // initialize decoder context + vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0); + } + vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0); + vpx_codec_iter_t iterator = 0; + vpx_image_t* image; + while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) { + depth.create(image->d_h, image->d_w, CV_8UC1); + uchar* yline = image->planes[0]; + uchar* vline = image->planes[1]; + const uchar EIGHT_BIT_MAXIMUM = 255; + const uchar MASK_THRESHOLD = 192; + for (int i = 0; i < image->d_h; i += 2) { + uchar* ysrc = yline; + uchar* vsrc = vline; + for (int j = 0; j < image->d_w; j += 2) { + if (*vsrc++ < MASK_THRESHOLD) { + *depth.ptr(i, j) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM; + + } else { + *depth.ptr(i, j) = ysrc[0]; + *depth.ptr(i, j + 1) = ysrc[1]; + *depth.ptr(i + 1, j) = ysrc[image->stride[0]]; + *depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1]; + } + ysrc += 2; + } + yline += image->stride[0] * 2; + vline += image->stride[1]; + } + } + } + QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color), + Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio)); } return dataBytes; @@ -368,23 +392,40 @@ void Face::cycleRenderMode() { } void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) { + if (color.empty()) { + // release our textures, if any; there's no more video + if (_colorTextureID != 0) { + glDeleteTextures(1, &_colorTextureID); + _colorTextureID = 0; + } + if (_depthTextureID != 0) { + glDeleteTextures(1, &_depthTextureID); + _depthTextureID = 0; + } + return; + } + if (_colorTextureID == 0) { glGenTextures(1, &_colorTextureID); - glBindTexture(GL_TEXTURE_2D, _colorTextureID); + } + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + bool recreateTextures = (_textureSize.width != color.cols || _textureSize.height != color.rows); + if (recreateTextures) { glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, color.cols, color.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); _textureSize = color.size(); _textureRect = RotatedRect(Point2f(color.cols * 0.5f, color.rows * 0.5f), _textureSize, 0.0f); - + } else { - glBindTexture(GL_TEXTURE_2D, _colorTextureID); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, color.cols, color.rows, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); } if (!depth.empty()) { if (_depthTextureID == 0) { glGenTextures(1, &_depthTextureID); - glBindTexture(GL_TEXTURE_2D, _depthTextureID); + } + glBindTexture(GL_TEXTURE_2D, _depthTextureID); + if (recreateTextures) { glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depth.cols, depth.rows, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); @@ -400,3 +441,13 @@ void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRati _aspectRatio = aspectRatio; } +void Face::destroyCodecs() { + if (_colorCodec.name != 0) { + vpx_codec_destroy(&_colorCodec); + _colorCodec.name = 0; + } + if (_depthCodec.name != 0) { + vpx_codec_destroy(&_depthCodec); + _depthCodec.name = 0; + } +} diff --git a/interface/src/avatar/Face.h b/interface/src/avatar/Face.h index a40111dcd3..893318f186 100644 --- a/interface/src/avatar/Face.h +++ b/interface/src/avatar/Face.h @@ -51,6 +51,8 @@ private: enum RenderMode { MESH, POINTS, RENDER_MODE_COUNT }; + void destroyCodecs(); + Head* _owningHead; RenderMode _renderMode; GLuint _colorTextureID; @@ -61,6 +63,7 @@ private: vpx_codec_ctx_t _colorCodec; vpx_codec_ctx_t _depthCodec; + bool _lastFullFrame; QByteArray _arrivingFrame; int _frameCount; From a0ba78867906748b1ae382c9fa3968aa4c17b7f4 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 1 Aug 2013 15:40:39 -0700 Subject: [PATCH 11/14] implement grid landscape import --- interface/src/Application.cpp | 180 +++++++++++++++++--------- libraries/voxels/src/VoxelConstants.h | 2 + voxel-server/src/main.cpp | 15 +++ 3 files changed, 136 insertions(+), 61 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f331ee9407..87f52ef285 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1490,16 +1490,18 @@ bool Application::sendVoxelsOperation(VoxelNode* node, void* extraData) { uint64_t now = usecTimestampNow(); // dynamically sleep until we need to fire off the next set of voxels - const uint64_t CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS = 1000 * 5; // 1 packet every 10 milliseconds uint64_t elapsed = now - args->lastSendTime; int usecToSleep = CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS - elapsed; if (usecToSleep > 0) { - qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", - args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed, usecToSleep); + //qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", + // args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed, usecToSleep); + + Application::getInstance()->timer(); + usleep(usecToSleep); } else { - qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", - args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed); + //qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", + // args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed); } args->lastSendTime = now; } @@ -1567,70 +1569,127 @@ void Application::importVoxelsToClipboard() { void Application::importVoxels() { QString desktopLocation = QDesktopServices::storageLocation(QDesktopServices::DesktopLocation); - QString fileNameString = QFileDialog::getOpenFileName(_glWidget, tr("Import Voxels"), desktopLocation, + + QStringList fileNameStringList = QFileDialog::getOpenFileNames(_glWidget, tr("Import Voxels"), desktopLocation, tr(IMPORT_FILE_TYPES)); - QByteArray fileNameAscii = fileNameString.toAscii(); - const char* fileName = fileNameAscii.data(); + + // remember the "selected" voxel point before we do any importing... + float originalX = _mouseVoxel.x; + float originalZ = _mouseVoxel.z; + + const int PNG_TYPE_NAME_LENGTH = 4; + const int SVO_TYPE_NAME_LENGTH = 4; + const int SCH_TYPE_NAME_LENGTH = 10; + + for (int i = 0; i < fileNameStringList.size(); i++) { + QString fileNameString = fileNameStringList.at(i); + QByteArray fileNameAscii = fileNameString.toLocal8Bit(); + const char* fileName = fileNameAscii.data(); - VoxelTree importVoxels; - if (fileNameString.endsWith(".png", Qt::CaseInsensitive)) { - QImage pngImage = QImage(fileName); - if (pngImage.height() != pngImage.width()) { - qDebug("ERROR: Bad PNG size: height != width.\n"); - return; + int fileTypeNameLength = 0; + VoxelTree importVoxels; + if (fileNameString.endsWith(".png", Qt::CaseInsensitive)) { + QImage pngImage = QImage(fileName); + fileTypeNameLength = PNG_TYPE_NAME_LENGTH; + if (pngImage.height() != pngImage.width()) { + qDebug("ERROR: Bad PNG size: height != width.\n"); + return; + } + + const uint32_t* pixels; + if (pngImage.format() == QImage::Format_ARGB32) { + pixels = reinterpret_cast(pngImage.constBits()); + } else { + QImage tmp = pngImage.convertToFormat(QImage::Format_ARGB32); + pixels = reinterpret_cast(tmp.constBits()); + } + + importVoxels.readFromSquareARGB32Pixels(pixels, pngImage.height()); + } else if (fileNameString.endsWith(".svo", Qt::CaseInsensitive)) { + importVoxels.readFromSVOFile(fileName); + fileTypeNameLength = SVO_TYPE_NAME_LENGTH; + } else if (fileNameString.endsWith(".schematic", Qt::CaseInsensitive)) { + importVoxels.readFromSchematicFile(fileName); + fileTypeNameLength = SCH_TYPE_NAME_LENGTH; } - const uint32_t* pixels; - if (pngImage.format() == QImage::Format_ARGB32) { - pixels = reinterpret_cast(pngImage.constBits()); + int indexOfFirstPeriod = fileNameString.indexOf('.'); + + QString fileCoord = fileNameString.mid(indexOfFirstPeriod + 1, + fileNameString.length() - indexOfFirstPeriod - fileTypeNameLength - 1); + + indexOfFirstPeriod = fileCoord.indexOf('.'); + QString columnNumString = fileCoord.right(fileCoord.length() - indexOfFirstPeriod - 1); + QString rowNumString = fileCoord.left(indexOfFirstPeriod); + + int columnNum = columnNumString.toFloat(); + int rowNum = rowNumString.toFloat(); + + qDebug("columnNum: %d\t rowNum: %d\n", columnNum, rowNum); + + _mouseVoxel.x = originalX + (columnNum - 1) * _mouseVoxel.s; + _mouseVoxel.z = originalZ + (rowNum - 1) * _mouseVoxel.s; + + VoxelNode* selectedNode = _voxels.getVoxelAt(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); + + // Recurse the Import Voxels tree, where everything is root relative, and send all the colored voxels to + // the server as an set voxel message, this will also rebase the voxels to the new location + unsigned char* calculatedOctCode = NULL; + SendVoxelsOperationArgs args; + args.lastSendTime = usecTimestampNow(); + args.packetsSent = 0; + args.bytesSent = 0; + + int numBytesPacketHeader = populateTypeAndVersion(args.messageBuffer, PACKET_TYPE_SET_VOXEL_DESTRUCTIVE); + + unsigned short int* sequenceAt = (unsigned short int*)&args.messageBuffer[numBytesPacketHeader]; + *sequenceAt = 0; + args.bufferInUse = numBytesPacketHeader + sizeof(unsigned short int); // set to command + sequence + + // we only need the selected voxel to get the newBaseOctCode, which we can actually calculate from the + // voxel size/position details. + if (selectedNode) { + args.newBaseOctCode = selectedNode->getOctalCode(); } else { - QImage tmp = pngImage.convertToFormat(QImage::Format_ARGB32); - pixels = reinterpret_cast(tmp.constBits()); + args.newBaseOctCode = calculatedOctCode = pointToVoxel(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); + } + + qDebug("column:%d, row:%d, voxel:%f,%f,%f,%f\n", columnNum, rowNum, _mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s ); + + // send the insert/paste of these voxels + importVoxels.recurseTreeWithOperation(sendVoxelsOperation, &args); + + // If we have voxels left in the packet, then send the packet + if (args.bufferInUse > (numBytesPacketHeader + sizeof(unsigned short int))) { + controlledBroadcastToNodes(args.messageBuffer, args.bufferInUse, & NODE_TYPE_VOXEL_SERVER, 1); + + + args.packetsSent++; + args.bytesSent += args.bufferInUse; + + uint64_t now = usecTimestampNow(); + // dynamically sleep until we need to fire off the next set of voxels + uint64_t elapsed = now - args.lastSendTime; + int usecToSleep = CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS - elapsed; + + if (usecToSleep > 0) { + //qDebug("after sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", + // args.packetsSent, (long long int)args.bytesSent, (long long int)elapsed, usecToSleep); + usleep(usecToSleep); + } else { + //qDebug("after sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", + // args.packetsSent, (long long int)args.bytesSent, (long long int)elapsed); + } + args.lastSendTime = now; } - importVoxels.readFromSquareARGB32Pixels(pixels, pngImage.height()); - } else if (fileNameString.endsWith(".svo", Qt::CaseInsensitive)) { - importVoxels.readFromSVOFile(fileName); - } else if (fileNameString.endsWith(".schematic", Qt::CaseInsensitive)) { - importVoxels.readFromSchematicFile(fileName); + if (calculatedOctCode) { + delete[] calculatedOctCode; + } + } - VoxelNode* selectedNode = _voxels.getVoxelAt(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); - - // Recurse the Import Voxels tree, where everything is root relative, and send all the colored voxels to - // the server as an set voxel message, this will also rebase the voxels to the new location - unsigned char* calculatedOctCode = NULL; - SendVoxelsOperationArgs args; - args.lastSendTime = usecTimestampNow(); - args.packetsSent = 0; - args.bytesSent = 0; - - int numBytesPacketHeader = populateTypeAndVersion(args.messageBuffer, PACKET_TYPE_SET_VOXEL_DESTRUCTIVE); - - unsigned short int* sequenceAt = (unsigned short int*)&args.messageBuffer[numBytesPacketHeader]; - *sequenceAt = 0; - args.bufferInUse = numBytesPacketHeader + sizeof(unsigned short int); // set to command + sequence - - // we only need the selected voxel to get the newBaseOctCode, which we can actually calculate from the - // voxel size/position details. - if (selectedNode) { - args.newBaseOctCode = selectedNode->getOctalCode(); - } else { - args.newBaseOctCode = calculatedOctCode = pointToVoxel(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); - } - - importVoxels.recurseTreeWithOperation(sendVoxelsOperation, &args); - - // If we have voxels left in the packet, then send the packet - if (args.bufferInUse > (numBytesPacketHeader + sizeof(unsigned short int))) { - controlledBroadcastToNodes(args.messageBuffer, args.bufferInUse, & NODE_TYPE_VOXEL_SERVER, 1); - } - - if (calculatedOctCode) { - delete[] calculatedOctCode; - } - // restore the main window's active state _window->activateWindow(); } @@ -3454,6 +3513,7 @@ void* Application::networkReceive(void* args) { } if (NodeList::getInstance()->getNodeSocket()->receive(&senderAddress, app->_incomingPacket, &bytesReceived)) { + app->_packetCount++; app->_bytesCount += bytesReceived; @@ -3498,13 +3558,11 @@ void* Application::networkReceive(void* args) { Node* voxelServer = NodeList::getInstance()->soloNodeOfType(NODE_TYPE_VOXEL_SERVER); if (voxelServer && socketMatch(voxelServer->getActiveSocket(), &senderAddress)) { voxelServer->lock(); - if (messageData[0] == PACKET_TYPE_ENVIRONMENT_DATA) { app->_environment.parseData(&senderAddress, messageData, messageLength); } else { app->_voxels.parseData(messageData, messageLength); } - voxelServer->unlock(); } } diff --git a/libraries/voxels/src/VoxelConstants.h b/libraries/voxels/src/VoxelConstants.h index 55dff0c69a..4b971e9515 100644 --- a/libraries/voxels/src/VoxelConstants.h +++ b/libraries/voxels/src/VoxelConstants.h @@ -41,4 +41,6 @@ const glBufferIndex GLBUFFER_INDEX_UNKNOWN = ULONG_MAX; const float SIXTY_FPS_IN_MILLISECONDS = 1000.0f / 60.0f; const float VIEW_CULLING_RATE_IN_MILLISECONDS = 1000.0f; // once a second is fine +const uint64_t CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS = 1000 * 5; // 1 packet every 50 milliseconds + #endif \ No newline at end of file diff --git a/voxel-server/src/main.cpp b/voxel-server/src/main.cpp index c009eec2a5..d902f5dfeb 100644 --- a/voxel-server/src/main.cpp +++ b/voxel-server/src/main.cpp @@ -61,6 +61,7 @@ bool wantColorRandomizer = false; bool debugVoxelSending = false; bool shouldShowAnimationDebug = false; bool displayVoxelStats = false; +bool debugVoxelReceiving = false; EnvironmentData environmentData[3]; @@ -426,6 +427,8 @@ void attachVoxelNodeDataToNode(Node* newNode) { } } +int receivedPacketCount = 0; + int main(int argc, const char * argv[]) { pthread_mutex_init(&::treeLock, NULL); @@ -455,6 +458,10 @@ int main(int argc, const char * argv[]) { ::debugVoxelSending = cmdOptionExists(argc, argv, DEBUG_VOXEL_SENDING); printf("debugVoxelSending=%s\n", debug::valueOf(::debugVoxelSending)); + const char* DEBUG_VOXEL_RECEIVING = "--debugVoxelReceiving"; + ::debugVoxelReceiving = cmdOptionExists(argc, argv, DEBUG_VOXEL_RECEIVING); + printf("debugVoxelReceiving=%s\n", debug::valueOf(::debugVoxelReceiving)); + const char* WANT_ANIMATION_DEBUG = "--shouldShowAnimationDebug"; ::shouldShowAnimationDebug = cmdOptionExists(argc, argv, WANT_ANIMATION_DEBUG); printf("shouldShowAnimationDebug=%s\n", debug::valueOf(::shouldShowAnimationDebug)); @@ -584,12 +591,20 @@ int main(int argc, const char * argv[]) { destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", ::shouldShowAnimationDebug); + ::receivedPacketCount++; + unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader))); if (::shouldShowAnimationDebug) { printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n", destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", receivedBytes,itemNumber); } + + if (::debugVoxelReceiving) { + printf("got %s - %d command from client receivedBytes=%ld itemNumber=%d\n", + destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", + ::receivedPacketCount, receivedBytes,itemNumber); + } int atByte = numBytesPacketHeader + sizeof(itemNumber); unsigned char* voxelData = (unsigned char*)&packetData[atByte]; while (atByte < receivedBytes) { From c038e2c9baff78fb266420b33fd612f201061aa8 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 1 Aug 2013 18:32:48 -0700 Subject: [PATCH 12/14] Wrong array index here, and finger needs to be flagged as active so that the avatar mixer will encode it. --- libraries/avatars/src/HandData.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp index 7bb6eebdc0..639b4adc0f 100755 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -83,7 +83,8 @@ void HandData::decodeRemoteData(const std::vector& fingerVectors) { palm.setRawPosition(fingerVectors[vectorIndex++]); palm.setRawNormal(fingerVectors[vectorIndex++]); for (size_t f = 0; f < NUM_FINGERS_PER_HAND; ++f) { - FingerData& finger = palm.getFingers()[i]; + FingerData& finger = palm.getFingers()[f]; + finger.setActive(true); finger.setRawTipPosition(fingerVectors[vectorIndex++]); finger.setRawRootPosition(fingerVectors[vectorIndex++]); } From 69cac690675377be51e1a7e001de2be277a98e38 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 1 Aug 2013 18:56:14 -0700 Subject: [PATCH 13/14] Go ahead and show the leap hand by default, but don't include the fingers if the palms are inactive (!) --- interface/src/avatar/Hand.cpp | 2 +- libraries/avatars/src/HandData.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 28b1af0603..f70d9aff13 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -14,7 +14,7 @@ #include "Util.h" #include "renderer/ProgramObject.h" -const bool SHOW_LEAP_HAND = false; +const bool SHOW_LEAP_HAND = true; using namespace std; diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp index 639b4adc0f..ebef2b1247 100755 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -53,6 +53,9 @@ void HandData::encodeRemoteData(std::vector& fingerVectors) { fingerVectors.clear(); for (size_t i = 0; i < getNumPalms(); ++i) { PalmData& palm = getPalms()[i]; + if (!palm.isActive()) { + continue; + } fingerVectors.push_back(palm.getRawPosition()); fingerVectors.push_back(palm.getRawNormal()); for (size_t f = 0; f < palm.getNumFingers(); ++f) { From 11664f94463a0bc4e73aa634a92a5af8930194a3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 2 Aug 2013 10:12:21 -0700 Subject: [PATCH 14/14] Have the particle system default to off. --- interface/src/Application.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f37241a720..0f87e2f063 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1816,7 +1816,6 @@ void Application::initMenu() { (_renderLookatIndicatorOn = renderMenu->addAction("Lookat Indicator"))->setCheckable(true); _renderLookatIndicatorOn->setChecked(true); (_renderParticleSystemOn = renderMenu->addAction("Particle System"))->setCheckable(true); - _renderParticleSystemOn->setChecked(true); (_manualFirstPerson = renderMenu->addAction( "First Person", this, SLOT(setRenderFirstPerson(bool)), Qt::Key_P))->setCheckable(true); (_manualThirdPerson = renderMenu->addAction(