From 5f514642305d1d7b617cca4b93aa753a32630270 Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Wed, 17 Jul 2013 17:15:19 -0700 Subject: [PATCH 01/26] Start clicking on voxels makes sound --- interface/src/Application.cpp | 37 +++++++++++++++++++++++++++++++---- interface/src/Application.h | 6 +++++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 582bc4fcb4..a87564cf2e 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -197,6 +197,8 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _pitchFromTouch(0.0f), _groundPlaneImpact(0.0f), _mousePressed(false), + _isHoverVoxel(false), + _isHoverVoxelSounding(false), _mouseVoxelScale(1.0f / 1024.0f), _justEditedVoxel(false), _paintOn(false), @@ -848,6 +850,15 @@ void Application::mousePressEvent(QMouseEvent* event) { _mouseVoxelDragging = _mouseVoxel; _mousePressed = true; maybeEditVoxelUnderCursor(); + if (_isHoverVoxel && !_isHoverVoxelSounding) { + _hoverVoxelOriginalColor[0] = _hoverVoxel.red; + _hoverVoxelOriginalColor[1] = _hoverVoxel.green; + _hoverVoxelOriginalColor[2] = _hoverVoxel.blue; + _hoverVoxelOriginalColor[3] = 1; + _audio.startCollisionSound(1.0, 220, 0.0, 0.999f); + qDebug("s = %f\n", _hoverVoxel.s); + _isHoverVoxelSounding = true; + } } else if (event->button() == Qt::RightButton && checkedVoxelModeAction() != 0) { deleteVoxelUnderCursor(); @@ -1927,7 +1938,27 @@ void Application::update(float deltaTime) { glm::vec3 myLookAtFromMouse(mouseRayOrigin + mouseRayDirection); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); } - + + // Find the voxel we are hovering over, and respond if clicked + float distance; + BoxFace face; + + // If we have clicked on a voxel, update it's color + if (_isHoverVoxelSounding) { + qDebug("clicking on voxel\n"); + VoxelNode* hoveredNode = _voxels.getVoxelAt(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s); + nodeColor clickColor = { 1, 0, 0, 1 }; + float bright = _audio.getCollisionSoundMagnitude(); + hoveredNode->setColor(clickColor); + if (bright < 0.01f) { + hoveredNode->setColor(_hoverVoxelOriginalColor); + _isHoverVoxelSounding = false; + } + } else { + // Check for a new hover voxel + _isHoverVoxel = _voxels.findRayIntersection(mouseRayOrigin, mouseRayDirection, _hoverVoxel, distance, face); + } + // If we are dragging on a voxel, add thrust according to the amount the mouse is dragging const float VOXEL_GRAB_THRUST = 0.0f; if (_mousePressed && (_mouseVoxel.s != 0)) { @@ -1953,8 +1984,6 @@ void Application::update(float deltaTime) { (fabs(_myAvatar.getVelocity().x) + fabs(_myAvatar.getVelocity().y) + fabs(_myAvatar.getVelocity().z)) / 3 < MAX_AVATAR_EDIT_VELOCITY) { - float distance; - BoxFace face; if (_voxels.findRayIntersection(mouseRayOrigin, mouseRayDirection, _mouseVoxel, distance, face)) { if (distance < MAX_VOXEL_EDIT_DISTANCE) { // find the nearest voxel with the desired scale @@ -2140,7 +2169,7 @@ void Application::update(float deltaTime) { } void Application::updateAvatar(float deltaTime) { - + // When head is rotated via touch/mouse look, slowly turn body to follow const float BODY_FOLLOW_HEAD_RATE = 0.5f; // update body yaw by body yaw delta diff --git a/interface/src/Application.h b/interface/src/Application.h index 9333808a27..1f65ea7586 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -358,8 +358,12 @@ private: glm::vec3 _voxelThrust; bool _mousePressed; // true if mouse has been pressed (clear when finished) + VoxelDetail _hoverVoxel; // Stuff about the voxel I am hovering or clicking + bool _isHoverVoxel; + bool _isHoverVoxelSounding; + nodeColor _hoverVoxelOriginalColor; - VoxelDetail _mouseVoxel; // details of the voxel under the mouse cursor + VoxelDetail _mouseVoxel; // details of the voxel to be edited float _mouseVoxelScale; // the scale for adding/removing voxels glm::vec3 _lastMouseVoxelPos; // the position of the last mouse voxel edit bool _justEditedVoxel; // set when we've just added/deleted/colored a voxel From 0b101699152dc48f4a63b2b644bb78fb2fd2e9df Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 18 Jul 2013 11:13:16 -0700 Subject: [PATCH 02/26] More audio collision on hover experiments --- interface/src/Application.cpp | 18 ++++++++++++++---- interface/src/Head.cpp | 4 ++-- interface/src/Physics.cpp | 4 ++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 2001ba5d71..0cff3f0966 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -855,8 +855,7 @@ void Application::mousePressEvent(QMouseEvent* event) { _hoverVoxelOriginalColor[1] = _hoverVoxel.green; _hoverVoxelOriginalColor[2] = _hoverVoxel.blue; _hoverVoxelOriginalColor[3] = 1; - _audio.startCollisionSound(1.0, 220, 0.0, 0.999f); - qDebug("s = %f\n", _hoverVoxel.s); + _audio.startCollisionSound(1.0, 14080 * _hoverVoxel.s * TREE_SCALE, 0.0, 0.999f); _isHoverVoxelSounding = true; } @@ -1945,10 +1944,11 @@ void Application::update(float deltaTime) { // If we have clicked on a voxel, update it's color if (_isHoverVoxelSounding) { - qDebug("clicking on voxel\n"); VoxelNode* hoveredNode = _voxels.getVoxelAt(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s); - nodeColor clickColor = { 1, 0, 0, 1 }; float bright = _audio.getCollisionSoundMagnitude(); + nodeColor clickColor = { 255 * bright + _hoverVoxelOriginalColor[0] * (1.f - bright), + _hoverVoxelOriginalColor[1] * (1.f - bright), + _hoverVoxelOriginalColor[2] * (1.f - bright), 1 }; hoveredNode->setColor(clickColor); if (bright < 0.01f) { hoveredNode->setColor(_hoverVoxelOriginalColor); @@ -1956,7 +1956,17 @@ void Application::update(float deltaTime) { } } else { // Check for a new hover voxel + glm::vec4 oldVoxel(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s); _isHoverVoxel = _voxels.findRayIntersection(mouseRayOrigin, mouseRayDirection, _hoverVoxel, distance, face); + if (_isHoverVoxel && glm::vec4(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s) != oldVoxel) { + //qDebug("bing! x,y,z = %f,%f,%f\n", _hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z); + _hoverVoxelOriginalColor[0] = _hoverVoxel.red; + _hoverVoxelOriginalColor[1] = _hoverVoxel.green; + _hoverVoxelOriginalColor[2] = _hoverVoxel.blue; + _hoverVoxelOriginalColor[3] = 1; + _audio.startCollisionSound(1.0, 14080 * _hoverVoxel.s * TREE_SCALE, 0.0, 0.992f); + _isHoverVoxelSounding = true; + } } // If we are dragging on a voxel, add thrust according to the amount the mouse is dragging diff --git a/interface/src/Head.cpp b/interface/src/Head.cpp index 84ae9fffe9..bd73753b70 100644 --- a/interface/src/Head.cpp +++ b/interface/src/Head.cpp @@ -227,8 +227,8 @@ void Head::simulate(float deltaTime, bool isMine) { const float CAMERA_FOLLOW_HEAD_RATE_MAX = 0.5f; const float CAMERA_FOLLOW_HEAD_RATE_RAMP_RATE = 1.05f; const float CAMERA_STOP_TOLERANCE_DEGREES = 0.1f; - const float CAMERA_PITCH_START_TOLERANCE_DEGREES = 10.0f; - const float CAMERA_YAW_START_TOLERANCE_DEGREES = 3.0f; + const float CAMERA_PITCH_START_TOLERANCE_DEGREES = 20.0f; + const float CAMERA_YAW_START_TOLERANCE_DEGREES = 10.0f; float cameraHeadAngleDifference = glm::length(glm::vec2(_pitch - _cameraPitch, _yaw - _cameraYaw)); if (_isCameraMoving) { _cameraFollowHeadRate = glm::clamp(_cameraFollowHeadRate * CAMERA_FOLLOW_HEAD_RATE_RAMP_RATE, diff --git a/interface/src/Physics.cpp b/interface/src/Physics.cpp index 31e64ccad3..101087b1ef 100644 --- a/interface/src/Physics.cpp +++ b/interface/src/Physics.cpp @@ -38,3 +38,7 @@ void applyDamping(float deltaTime, glm::vec3& velocity, float linearStrength, fl } } +void applyDampedSpring(float deltaTime, glm::vec3& velocity, glm::vec3& position, glm::vec3& targetPosition, float k, float damping) { + +} + From 4af92d46f30040288452003e2b3b82e55d618320 Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 18 Jul 2013 16:50:37 -0700 Subject: [PATCH 03/26] Set lookAt to hovered voxel --- interface/src/Application.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 0cff3f0966..0b4b36eda1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1933,7 +1933,12 @@ void Application::update(float deltaTime) { // If the mouse is over another avatar's head... glm::vec3 myLookAtFromMouse(eyePosition); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); + } else if (_isHoverVoxel) { + // Look at the hovered voxel + glm::vec3 lookAtSpot = getMouseVoxelWorldCoordinates(_hoverVoxel); + _myAvatar.getHead().setLookAtPosition(lookAtSpot); } else { + // Just look in direction of the mouse ray glm::vec3 myLookAtFromMouse(mouseRayOrigin + mouseRayDirection); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); } From 729158f8824c8002921f23fd2e5584e45f66b968 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 31 Jul 2013 12:19:14 -0700 Subject: [PATCH 04/26] Added depth smoothing, experimenting with using depth minimum rather than mean. --- interface/src/Webcam.cpp | 30 ++++++++++++++++++++---------- interface/src/Webcam.h | 1 + 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index d4fa015ba0..032b483082 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -620,25 +620,35 @@ void FrameGrabber::grabFrame() { _faceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST); - // find the mean of the valid values - qint64 depthTotal = 0; - qint64 depthSamples = 0; - ushort* src = _faceDepth.ptr(); + _smoothedFaceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); + + // smooth and find the minimum/mean of the valid values const ushort ELEVEN_BIT_MINIMUM = 0; const ushort ELEVEN_BIT_MAXIMUM = 2047; + const float DEPTH_SMOOTHING = 0.25f; + qint64 depthTotal = 0; + qint64 depthSamples = 0; + ushort depthMinimum = ELEVEN_BIT_MAXIMUM; + ushort* src = _faceDepth.ptr(); + ushort* dest = _smoothedFaceDepth.ptr(); for (int i = 0; i < ENCODED_FACE_HEIGHT; i++) { for (int j = 0; j < ENCODED_FACE_WIDTH; j++) { ushort depth = *src++; if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { depthTotal += depth; + depthMinimum = min(depthMinimum, depth); depthSamples++; + + *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); } + dest++; } } - float mean = (depthSamples == 0) ? UNINITIALIZED_FACE_DEPTH : depthTotal / (float)depthSamples; + const ushort DEPTH_MINIMUM_OFFSET = 64; + float mean = (depthSamples == 0) ? UNINITIALIZED_FACE_DEPTH : depthMinimum + DEPTH_MINIMUM_OFFSET; // smooth the mean over time - const float DEPTH_OFFSET_SMOOTHING = 0.95f; + const float DEPTH_OFFSET_SMOOTHING = 0.5f; _smoothedMeanFaceDepth = (_smoothedMeanFaceDepth == UNINITIALIZED_FACE_DEPTH) ? mean : glm::mix(mean, _smoothedMeanFaceDepth, DEPTH_OFFSET_SMOOTHING); @@ -657,10 +667,10 @@ void FrameGrabber::grabFrame() { uchar* vdest = vline; uchar* udest = uline; for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { - ushort tl = *_faceDepth.ptr(i, j); - ushort tr = *_faceDepth.ptr(i, j + 1); - ushort bl = *_faceDepth.ptr(i + 1, j); - ushort br = *_faceDepth.ptr(i + 1, j + 1); + ushort tl = *_smoothedFaceDepth.ptr(i, j); + ushort tr = *_smoothedFaceDepth.ptr(i, j + 1); + ushort bl = *_smoothedFaceDepth.ptr(i + 1, j); + ushort br = *_smoothedFaceDepth.ptr(i + 1, j + 1); uchar mask = EIGHT_BIT_MAXIMUM; diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index 6c6d250897..077e9eed98 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -125,6 +125,7 @@ private: int _frameCount; cv::Mat _faceColor; cv::Mat _faceDepth; + cv::Mat _smoothedFaceDepth; QByteArray _encodedFace; cv::RotatedRect _smoothedFaceRect; From 415949cc26551be3b2d7127c2bb0c6b471c8bb4c Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 31 Jul 2013 14:40:33 -0700 Subject: [PATCH 05/26] Terminology change to reflect the fact that we're no longer using the mean. --- interface/src/Webcam.cpp | 41 ++++++++++++++++++---------------------- interface/src/Webcam.h | 4 ++-- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index 032b483082..3457064733 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -154,7 +154,7 @@ Webcam::~Webcam() { const float METERS_PER_MM = 1.0f / 1000.0f; -void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float meanFaceDepth, +void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midFaceDepth, const RotatedRect& faceRect, const JointVector& joints) { IplImage colorImage = color; glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); @@ -242,18 +242,18 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean if (_initialFaceRect.size.area() == 0) { _initialFaceRect = _faceRect; _estimatedPosition = glm::vec3(); - _initialFaceDepth = meanFaceDepth; + _initialFaceDepth = midFaceDepth; } else { float proportion, z; - if (meanFaceDepth == UNINITIALIZED_FACE_DEPTH) { + if (midFaceDepth == UNINITIALIZED_FACE_DEPTH) { proportion = sqrtf(_initialFaceRect.size.area() / (float)_faceRect.size.area()); const float INITIAL_DISTANCE_TO_CAMERA = 0.333f; z = INITIAL_DISTANCE_TO_CAMERA * proportion - INITIAL_DISTANCE_TO_CAMERA; } else { - z = (meanFaceDepth - _initialFaceDepth) * METERS_PER_MM; - proportion = meanFaceDepth / _initialFaceDepth; + z = (midFaceDepth - _initialFaceDepth) * METERS_PER_MM; + proportion = midFaceDepth / _initialFaceDepth; } const float POSITION_SCALE = 0.5f; _estimatedPosition = glm::vec3( @@ -271,7 +271,7 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean } FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), - _smoothedMeanFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) { + _smoothedMidFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) { } FrameGrabber::~FrameGrabber() { @@ -494,7 +494,7 @@ void FrameGrabber::grabFrame() { Rect imageBounds(0, 0, color.cols, color.rows); _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); } - + const int ENCODED_FACE_WIDTH = 128; const int ENCODED_FACE_HEIGHT = 128; if (_colorCodec.name == 0) { @@ -622,39 +622,34 @@ void FrameGrabber::grabFrame() { _smoothedFaceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); - // smooth and find the minimum/mean of the valid values + // smooth the depth over time const ushort ELEVEN_BIT_MINIMUM = 0; const ushort ELEVEN_BIT_MAXIMUM = 2047; const float DEPTH_SMOOTHING = 0.25f; - qint64 depthTotal = 0; - qint64 depthSamples = 0; - ushort depthMinimum = ELEVEN_BIT_MAXIMUM; ushort* src = _faceDepth.ptr(); ushort* dest = _smoothedFaceDepth.ptr(); + ushort minimumDepth = numeric_limits::max(); for (int i = 0; i < ENCODED_FACE_HEIGHT; i++) { for (int j = 0; j < ENCODED_FACE_WIDTH; j++) { ushort depth = *src++; if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { - depthTotal += depth; - depthMinimum = min(depthMinimum, depth); - depthSamples++; - + minimumDepth = min(minimumDepth, depth); *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); } dest++; } } - const ushort DEPTH_MINIMUM_OFFSET = 64; - float mean = (depthSamples == 0) ? UNINITIALIZED_FACE_DEPTH : depthMinimum + DEPTH_MINIMUM_OFFSET; + const ushort MINIMUM_DEPTH_OFFSET = 64; + float midFaceDepth = minimumDepth + MINIMUM_DEPTH_OFFSET; - // smooth the mean over time - const float DEPTH_OFFSET_SMOOTHING = 0.5f; - _smoothedMeanFaceDepth = (_smoothedMeanFaceDepth == UNINITIALIZED_FACE_DEPTH) ? mean : - glm::mix(mean, _smoothedMeanFaceDepth, DEPTH_OFFSET_SMOOTHING); + // smooth the mid face depth over time + const float MID_FACE_DEPTH_SMOOTHING = 0.5f; + _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : + glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); // convert from 11 to 8 bits for preview/local display const uchar EIGHT_BIT_MIDPOINT = 128; - double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMeanFaceDepth; + double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); // likewise for the encoded representation @@ -707,7 +702,7 @@ void FrameGrabber::grabFrame() { Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", - Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMeanFaceDepth), + Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); } diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index 077e9eed98..021ef3bc15 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -62,7 +62,7 @@ public: public slots: void setEnabled(bool enabled); - void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float meanFaceDepth, + void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth, const cv::RotatedRect& faceRect, const JointVector& joints); private: @@ -118,7 +118,7 @@ private: cv::Mat _backProject; cv::Rect _searchWindow; cv::Mat _grayDepthFrame; - float _smoothedMeanFaceDepth; + float _smoothedMidFaceDepth; vpx_codec_ctx_t _colorCodec; vpx_codec_ctx_t _depthCodec; From d18a9dc49905eeffa899c370a548684e15bb3bba Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 31 Jul 2013 17:06:09 -0700 Subject: [PATCH 06/26] Working on full frame video support. --- interface/src/Webcam.cpp | 405 +++++++++++++++++++++------------------ interface/src/Webcam.h | 4 + 2 files changed, 224 insertions(+), 185 deletions(-) diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index 3457064733..b11f134fc5 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -270,7 +270,7 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame())); } -FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), +FrameGrabber::FrameGrabber() : _initialized(false), _videoSendMode(FULL_FRAME_VIDEO), _capture(0), _searchWindow(0, 0, 0, 0), _smoothedMidFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) { } @@ -364,6 +364,11 @@ static void XN_CALLBACK_TYPE calibrationCompleted(SkeletonCapability& capability } #endif +void FrameGrabber::cycleVideoSendMode() { + _videoSendMode = (VideoSendMode)((_videoSendMode + 1) % VIDEO_SEND_MODE_COUNT); + _searchWindow = cv::Rect(0, 0, 0, 0); +} + void FrameGrabber::reset() { _searchWindow = cv::Rect(0, 0, 0, 0); @@ -462,56 +467,54 @@ void FrameGrabber::grabFrame() { color = image; } - // if we don't have a search window (yet), try using the face cascade - int channels = 0; - float ranges[] = { 0, 180 }; - const float* range = ranges; - if (_searchWindow.area() == 0) { - vector faces; - _faceCascade.detectMultiScale(color, faces, 1.1, 6); - if (!faces.empty()) { - _searchWindow = faces.front(); - updateHSVFrame(color, format); - - Mat faceHsv(_hsvFrame, _searchWindow); - Mat faceMask(_mask, _searchWindow); - int sizes = 30; - calcHist(&faceHsv, 1, &channels, faceMask, _histogram, 1, &sizes, &range); - double min, max; - minMaxLoc(_histogram, &min, &max); - _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max); - } - } RotatedRect faceRect; - if (_searchWindow.area() > 0) { - updateHSVFrame(color, format); + int encodedWidth; + int encodedHeight; + int depthBitrateMultiplier = 1; + if (_videoSendMode == FULL_FRAME_VIDEO) { + // no need to find the face if we're sending full frame video + faceRect.center = Point2f(color.cols / 2.0f, color.rows / 2.0f); + faceRect.size = Size2f(color.cols, color.rows); + encodedWidth = color.cols; + encodedHeight = color.rows; - calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range); - bitwise_and(_backProject, _mask, _backProject); - - faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1)); - Rect faceBounds = faceRect.boundingRect(); - Rect imageBounds(0, 0, color.cols, color.rows); - _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); - } - - const int ENCODED_FACE_WIDTH = 128; - const int ENCODED_FACE_HEIGHT = 128; - if (_colorCodec.name == 0) { - // initialize encoder context(s) - vpx_codec_enc_cfg_t codecConfig; - vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0); - codecConfig.rc_target_bitrate = ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * - codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h; - codecConfig.g_w = ENCODED_FACE_WIDTH; - codecConfig.g_h = ENCODED_FACE_HEIGHT; - vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0); - - if (!depth.empty()) { - int DEPTH_BITRATE_MULTIPLIER = 2; - codecConfig.rc_target_bitrate *= 2; - vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0); + } else { + // if we don't have a search window (yet), try using the face cascade + int channels = 0; + float ranges[] = { 0, 180 }; + const float* range = ranges; + if (_searchWindow.area() == 0) { + vector faces; + _faceCascade.detectMultiScale(color, faces, 1.1, 6); + if (!faces.empty()) { + _searchWindow = faces.front(); + updateHSVFrame(color, format); + + Mat faceHsv(_hsvFrame, _searchWindow); + Mat faceMask(_mask, _searchWindow); + int sizes = 30; + calcHist(&faceHsv, 1, &channels, faceMask, _histogram, 1, &sizes, &range); + double min, max; + minMaxLoc(_histogram, &min, &max); + _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max); + } } + if (_searchWindow.area() > 0) { + updateHSVFrame(color, format); + + calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range); + bitwise_and(_backProject, _mask, _backProject); + + faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1)); + Rect faceBounds = faceRect.boundingRect(); + Rect imageBounds(0, 0, color.cols, color.rows); + _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); + } + const int ENCODED_FACE_WIDTH = 128; + const int ENCODED_FACE_HEIGHT = 128; + encodedWidth = ENCODED_FACE_WIDTH; + encodedHeight = ENCODED_FACE_HEIGHT; + depthBitrateMultiplier = 2; } // correct for 180 degree rotations @@ -535,150 +538,86 @@ void FrameGrabber::grabFrame() { _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); } - // resize/rotate face into encoding rectangle - _faceColor.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_8UC3); - Point2f sourcePoints[4]; - _smoothedFaceRect.points(sourcePoints); - Point2f destPoints[] = { Point2f(0, ENCODED_FACE_HEIGHT), Point2f(0, 0), Point2f(ENCODED_FACE_WIDTH, 0) }; - Mat transform = getAffineTransform(sourcePoints, destPoints); - warpAffine(color, _faceColor, transform, _faceColor.size()); - - // convert from RGB to YV12 - const int ENCODED_BITS_PER_Y = 8; - const int ENCODED_BITS_PER_VU = 2; - const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU; - const int BITS_PER_BYTE = 8; - _encodedFace.resize(ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE); - vpx_image_t vpxImage; - vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, 1, (unsigned char*)_encodedFace.data()); - uchar* yline = vpxImage.planes[0]; - uchar* vline = vpxImage.planes[1]; - uchar* uline = vpxImage.planes[2]; - const int Y_RED_WEIGHT = (int)(0.299 * 256); - const int Y_GREEN_WEIGHT = (int)(0.587 * 256); - const int Y_BLUE_WEIGHT = (int)(0.114 * 256); - const int V_RED_WEIGHT = (int)(0.713 * 256); - const int U_BLUE_WEIGHT = (int)(0.564 * 256); - int redIndex = 0; - int greenIndex = 1; - int blueIndex = 2; - if (format == GL_BGR) { - redIndex = 2; - blueIndex = 0; - } - for (int i = 0; i < ENCODED_FACE_HEIGHT; i += 2) { - uchar* ydest = yline; - uchar* vdest = vline; - uchar* udest = uline; - for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { - uchar* tl = _faceColor.ptr(i, j); - uchar* tr = _faceColor.ptr(i, j + 1); - uchar* bl = _faceColor.ptr(i + 1, j); - uchar* br = _faceColor.ptr(i + 1, j + 1); + if (_videoSendMode != NO_VIDEO) { + if (_colorCodec.name == 0) { + // initialize encoder context(s) + vpx_codec_enc_cfg_t codecConfig; + vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0); + codecConfig.rc_target_bitrate = encodedWidth * encodedHeight * + codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h; + codecConfig.g_w = encodedWidth; + codecConfig.g_h = encodedHeight; + vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0); - ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] * - Y_GREEN_WEIGHT + bl[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] * - Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest += 2; - - int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex]; - int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex]; - int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex]; - int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8; - - *vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128; - *udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128; - } - yline += vpxImage.stride[0] * 2; - vline += vpxImage.stride[1]; - uline += vpxImage.stride[2]; - } - - // encode the frame - vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); - - // start the payload off with the aspect ratio - QByteArray payload(sizeof(float), 0); - *(float*)payload.data() = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; - - // extract the encoded frame - vpx_codec_iter_t iterator = 0; - const vpx_codec_cx_pkt_t* packet; - while ((packet = vpx_codec_get_cx_data(&_colorCodec, &iterator)) != 0) { - if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { - // prepend the length, which will indicate whether there's a depth frame too - payload.append((const char*)&packet->data.frame.sz, sizeof(packet->data.frame.sz)); - payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); - } - } - - if (!depth.empty()) { - // warp the face depth without interpolation (because it will contain invalid zero values) - _faceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); - warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST); - - _smoothedFaceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); - - // smooth the depth over time - const ushort ELEVEN_BIT_MINIMUM = 0; - const ushort ELEVEN_BIT_MAXIMUM = 2047; - const float DEPTH_SMOOTHING = 0.25f; - ushort* src = _faceDepth.ptr(); - ushort* dest = _smoothedFaceDepth.ptr(); - ushort minimumDepth = numeric_limits::max(); - for (int i = 0; i < ENCODED_FACE_HEIGHT; i++) { - for (int j = 0; j < ENCODED_FACE_WIDTH; j++) { - ushort depth = *src++; - if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { - minimumDepth = min(minimumDepth, depth); - *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); - } - dest++; + if (!depth.empty()) { + codecConfig.rc_target_bitrate *= depthBitrateMultiplier; + vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0); } } - const ushort MINIMUM_DEPTH_OFFSET = 64; - float midFaceDepth = minimumDepth + MINIMUM_DEPTH_OFFSET; + + Mat transform; + if (_videoSendMode == FACE_VIDEO) { + // resize/rotate face into encoding rectangle + _faceColor.create(encodedHeight, encodedWidth, CV_8UC3); + Point2f sourcePoints[4]; + _smoothedFaceRect.points(sourcePoints); + Point2f destPoints[] = { Point2f(0, encodedHeight), Point2f(0, 0), Point2f(encodedWidth, 0) }; + transform = getAffineTransform(sourcePoints, destPoints); + warpAffine(color, _faceColor, transform, _faceColor.size()); - // smooth the mid face depth over time - const float MID_FACE_DEPTH_SMOOTHING = 0.5f; - _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : - glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); - - // convert from 11 to 8 bits for preview/local display - const uchar EIGHT_BIT_MIDPOINT = 128; - double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; - depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); - - // likewise for the encoded representation + } else { + _faceColor = color; + } + + // convert from RGB to YV12 + const int ENCODED_BITS_PER_Y = 8; + const int ENCODED_BITS_PER_VU = 2; + const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU; + const int BITS_PER_BYTE = 8; + _encodedFace.resize(encodedWidth * encodedHeight * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE); + vpx_image_t vpxImage; + vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, encodedWidth, encodedHeight, 1, + (unsigned char*)_encodedFace.data()); uchar* yline = vpxImage.planes[0]; uchar* vline = vpxImage.planes[1]; uchar* uline = vpxImage.planes[2]; - const uchar EIGHT_BIT_MAXIMUM = 255; - for (int i = 0; i < ENCODED_FACE_HEIGHT; i += 2) { + const int Y_RED_WEIGHT = (int)(0.299 * 256); + const int Y_GREEN_WEIGHT = (int)(0.587 * 256); + const int Y_BLUE_WEIGHT = (int)(0.114 * 256); + const int V_RED_WEIGHT = (int)(0.713 * 256); + const int U_BLUE_WEIGHT = (int)(0.564 * 256); + int redIndex = 0; + int greenIndex = 1; + int blueIndex = 2; + if (format == GL_BGR) { + redIndex = 2; + blueIndex = 0; + } + for (int i = 0; i < encodedHeight; i += 2) { uchar* ydest = yline; uchar* vdest = vline; uchar* udest = uline; - for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { - ushort tl = *_smoothedFaceDepth.ptr(i, j); - ushort tr = *_smoothedFaceDepth.ptr(i, j + 1); - ushort bl = *_smoothedFaceDepth.ptr(i + 1, j); - ushort br = *_smoothedFaceDepth.ptr(i + 1, j + 1); - - uchar mask = EIGHT_BIT_MAXIMUM; + for (int j = 0; j < encodedWidth; j += 2) { + uchar* tl = _faceColor.ptr(i, j); + uchar* tr = _faceColor.ptr(i, j + 1); + uchar* bl = _faceColor.ptr(i + 1, j); + uchar* br = _faceColor.ptr(i + 1, j + 1); - ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(tl + depthOffset); - ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(tr + depthOffset); - ydest[vpxImage.stride[0]] = (bl == ELEVEN_BIT_MINIMUM) ? - (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(bl + depthOffset); - ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? - (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(br + depthOffset); + ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] * + Y_GREEN_WEIGHT + bl[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] * + Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8; ydest += 2; - - *vdest++ = mask; - *udest++ = EIGHT_BIT_MIDPOINT; + + int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex]; + int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex]; + int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex]; + int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8; + + *vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128; + *udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128; } yline += vpxImage.stride[0] * 2; vline += vpxImage.stride[1]; @@ -686,21 +625,117 @@ void FrameGrabber::grabFrame() { } // encode the frame - vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME); + vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); + + // start the payload off with the aspect ratio + QByteArray payload(sizeof(float), 0); + *(float*)payload.data() = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; // extract the encoded frame vpx_codec_iter_t iterator = 0; const vpx_codec_cx_pkt_t* packet; - while ((packet = vpx_codec_get_cx_data(&_depthCodec, &iterator)) != 0) { + while ((packet = vpx_codec_get_cx_data(&_colorCodec, &iterator)) != 0) { if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { + // prepend the length, which will indicate whether there's a depth frame too + payload.append((const char*)&packet->data.frame.sz, sizeof(packet->data.frame.sz)); payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); } } + + if (!depth.empty()) { + if (_videoSendMode == FACE_VIDEO) { + // warp the face depth without interpolation (because it will contain invalid zero values) + _faceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST); + + } else { + _faceDepth = depth; + } + _smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + + // smooth the depth over time + const ushort ELEVEN_BIT_MINIMUM = 0; + const ushort ELEVEN_BIT_MAXIMUM = 2047; + const float DEPTH_SMOOTHING = 0.25f; + ushort* src = _faceDepth.ptr(); + ushort* dest = _smoothedFaceDepth.ptr(); + ushort minimumDepth = numeric_limits::max(); + for (int i = 0; i < encodedHeight; i++) { + for (int j = 0; j < encodedWidth; j++) { + ushort depth = *src++; + if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { + minimumDepth = min(minimumDepth, depth); + *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); + } + dest++; + } + } + const ushort MINIMUM_DEPTH_OFFSET = 64; + const float FIXED_MID_DEPTH = 640.0f; + float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH; + + // smooth the mid face depth over time + const float MID_FACE_DEPTH_SMOOTHING = 0.5f; + _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : + glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); + + // convert from 11 to 8 bits for preview/local display + const uchar EIGHT_BIT_MIDPOINT = 128; + double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; + depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); + + // likewise for the encoded representation + uchar* yline = vpxImage.planes[0]; + uchar* vline = vpxImage.planes[1]; + uchar* uline = vpxImage.planes[2]; + const uchar EIGHT_BIT_MAXIMUM = 255; + for (int i = 0; i < encodedHeight; i += 2) { + uchar* ydest = yline; + uchar* vdest = vline; + uchar* udest = uline; + for (int j = 0; j < encodedWidth; j += 2) { + ushort tl = *_smoothedFaceDepth.ptr(i, j); + ushort tr = *_smoothedFaceDepth.ptr(i, j + 1); + ushort bl = *_smoothedFaceDepth.ptr(i + 1, j); + ushort br = *_smoothedFaceDepth.ptr(i + 1, j + 1); + + uchar mask = EIGHT_BIT_MAXIMUM; + + ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(tl + depthOffset); + ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(tr + depthOffset); + ydest[vpxImage.stride[0]] = (bl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(bl + depthOffset); + ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(br + depthOffset); + ydest += 2; + + *vdest++ = mask; + *udest++ = EIGHT_BIT_MIDPOINT; + } + yline += vpxImage.stride[0] * 2; + vline += vpxImage.stride[1]; + uline += vpxImage.stride[2]; + } + + // encode the frame + vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME); + + // extract the encoded frame + vpx_codec_iter_t iterator = 0; + const vpx_codec_cx_pkt_t* packet; + while ((packet = vpx_codec_get_cx_data(&_depthCodec, &iterator)) != 0) { + if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { + payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); + } + } + } + + QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", + Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); } - QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", - Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); - QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index 021ef3bc15..b5a26d9b13 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -100,16 +100,20 @@ public: public slots: + void cycleVideoSendMode(); void reset(); void shutdown(); void grabFrame(); private: + enum VideoSendMode { NO_VIDEO, FACE_VIDEO, FULL_FRAME_VIDEO, VIDEO_SEND_MODE_COUNT }; + bool init(); void updateHSVFrame(const cv::Mat& frame, int format); bool _initialized; + VideoSendMode _videoSendMode; CvCapture* _capture; cv::CascadeClassifier _faceCascade; cv::Mat _hsvFrame; From 5b2693f99c9373f0be2e1a581cf16370e32418c9 Mon Sep 17 00:00:00 2001 From: Jeffrey Ventrella Date: Wed, 31 Jul 2013 17:59:15 -0700 Subject: [PATCH 07/26] added emitter active bool for switching emitters on/off when Leap loses fingers --- interface/src/ParticleSystem.cpp | 53 ++++++++++++++------------ interface/src/ParticleSystem.h | 5 ++- interface/src/avatar/Hand.cpp | 64 +++++++++++++++++++++----------- 3 files changed, 77 insertions(+), 45 deletions(-) diff --git a/interface/src/ParticleSystem.cpp b/interface/src/ParticleSystem.cpp index 5e2b92bb64..fcea8ca270 100644 --- a/interface/src/ParticleSystem.cpp +++ b/interface/src/ParticleSystem.cpp @@ -27,6 +27,7 @@ ParticleSystem::ParticleSystem() { for (unsigned int emitterIndex = 0; emitterIndex < MAX_EMITTERS; emitterIndex++) { Emitter * e = &_emitter[emitterIndex]; + e->active = false; e->position = glm::vec3(0.0f, 0.0f, 0.0f); e->previousPosition = glm::vec3(0.0f, 0.0f, 0.0f); e->direction = glm::vec3(0.0f, 1.0f, 0.0f); @@ -72,25 +73,16 @@ void ParticleSystem::simulate(float deltaTime) { _timer += deltaTime; - // emit particles - for (int e = 0; e < _numEmitters; e++) { + // update emitters + for (int emitterIndex = 0; emitterIndex < _numEmitters; emitterIndex++) { + assert(emitterIndex <= MAX_EMITTERS); - assert(e >= 0); - assert(e <= MAX_EMITTERS); - assert(_emitter[e].rate >= 0); - - _emitter[e].emitReserve += _emitter[e].rate * deltaTime; - _emitter[e].numParticlesEmittedThisTime = (int)_emitter[e].emitReserve; - _emitter[e].emitReserve -= _emitter[e].numParticlesEmittedThisTime; - - for (int p = 0; p < _emitter[e].numParticlesEmittedThisTime; p++) { - float timeFraction = (float)p / (float)_emitter[e].numParticlesEmittedThisTime; - createParticle(e, timeFraction); + if (_emitter[emitterIndex].active) { + updateEmitter(emitterIndex, deltaTime); } } - - // update particles + // update particles for (int p = 0; p < MAX_PARTICLES; p++) { if (_particle[p].alive) { if (_particle[p].age > _emitter[_particle[p].emitterIndex].particleLifespan) { @@ -102,6 +94,20 @@ void ParticleSystem::simulate(float deltaTime) { } } + +void ParticleSystem::updateEmitter(int emitterIndex, float deltaTime) { + + _emitter[emitterIndex].emitReserve += _emitter[emitterIndex].rate * deltaTime; + _emitter[emitterIndex].numParticlesEmittedThisTime = (int)_emitter[emitterIndex].emitReserve; + _emitter[emitterIndex].emitReserve -= _emitter[emitterIndex].numParticlesEmittedThisTime; + + for (int p = 0; p < _emitter[emitterIndex].numParticlesEmittedThisTime; p++) { + float timeFraction = (float)p / (float)_emitter[emitterIndex].numParticlesEmittedThisTime; + createParticle(emitterIndex, timeFraction); + } +} + + void ParticleSystem::createParticle(int e, float timeFraction) { for (unsigned int p = 0; p < MAX_PARTICLES; p++) { @@ -212,7 +218,6 @@ void ParticleSystem::setParticleAttributes(int emitterIndex, ParticleLifeStage l } - void ParticleSystem::updateParticle(int p, float deltaTime) { Emitter myEmitter = _emitter[_particle[p].emitterIndex]; @@ -363,14 +368,16 @@ void ParticleSystem::killAllParticles() { void ParticleSystem::render() { // render the emitters - for (int e = 0; e < _numEmitters; e++) { + for (int e = 0; e < MAX_EMITTERS; e++) { - if (_emitter[e].showingBaseParticle) { - glColor4f(_particle[0].color.r, _particle[0].color.g, _particle[0].color.b, _particle[0].color.a); - glPushMatrix(); - glTranslatef(_emitter[e].position.x, _emitter[e].position.y, _emitter[e].position.z); - glutSolidSphere(_particle[0].radius, _emitter[e].particleResolution, _emitter[e].particleResolution); - glPopMatrix(); + if (_emitter[e].active) { + if (_emitter[e].showingBaseParticle) { + glColor4f(_particle[0].color.r, _particle[0].color.g, _particle[0].color.b, _particle[0].color.a); + glPushMatrix(); + glTranslatef(_emitter[e].position.x, _emitter[e].position.y, _emitter[e].position.z); + glutSolidSphere(_particle[0].radius, _emitter[e].particleResolution, _emitter[e].particleResolution); + glPopMatrix(); + } } if (_emitter[e].visible) { diff --git a/interface/src/ParticleSystem.h b/interface/src/ParticleSystem.h index d79f621f69..a0131883a1 100644 --- a/interface/src/ParticleSystem.h +++ b/interface/src/ParticleSystem.h @@ -10,10 +10,10 @@ #include -const int MAX_PARTICLES = 5000; const int NULL_EMITTER = -1; const int NULL_PARTICLE = -1; const int MAX_EMITTERS = 100; +const int MAX_PARTICLES = 5000; enum ParticleRenderStyle { @@ -78,6 +78,7 @@ public: void setParticleAttributes (int emitterIndex, ParticleAttributes attributes); // set attributes for whole life of particles void setParticleAttributes (int emitterIndex, ParticleLifeStage lifeStage, ParticleAttributes attributes); // set attributes for this life stage void setEmitterPosition (int emitterIndex, glm::vec3 position ); + void setEmitterActive (int emitterIndex, bool active ) {_emitter[emitterIndex].active = active; } void setEmitterParticleResolution (int emitterIndex, int resolution ) {_emitter[emitterIndex].particleResolution = resolution; } void setEmitterDirection (int emitterIndex, glm::vec3 direction ) {_emitter[emitterIndex].direction = direction; } void setShowingEmitter (int emitterIndex, bool showing ) {_emitter[emitterIndex].visible = showing; } @@ -101,6 +102,7 @@ private: }; struct Emitter { + bool active; // if false, the emitter is disabled - allows for easy switching on and off glm::vec3 position; // the position of the emitter in world coordinates glm::vec3 previousPosition; // the position of the emitter in the previous time step glm::vec3 direction; // a normalized vector used as an axis for particle emission and other effects @@ -124,6 +126,7 @@ private: float _timer; // private methods + void updateEmitter(int emitterIndex, float deltaTime); void updateParticle(int index, float deltaTime); void createParticle(int e, float timeFraction); void killParticle(int p); diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 28b1af0603..3eec1925c0 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -244,6 +244,7 @@ void Hand::setLeapHands(const std::vector& handPositions, } } + // call this right after the geometry of the leap hands are set void Hand::updateRaveGloveEmitters() { @@ -253,7 +254,7 @@ void Hand::updateRaveGloveEmitters() { if(debug) printf( "\n" ); if(debug) printf( "------------------------------------\n" ); - if(debug) printf( "updating rave glove emitters:\n" ); + if(debug) printf( "updating rave glove emitters: \n" ); if(debug) printf( "------------------------------------\n" ); int emitterIndex = 0; @@ -271,16 +272,16 @@ void Hand::updateRaveGloveEmitters() { for (size_t f = 0; f < palm.getNumFingers(); ++f) { FingerData& finger = palm.getFingers()[f]; - if(debug) printf( "emitterIndex %d: ", emitterIndex ); + if(debug) printf( "emitterIndex %d: ", emitterIndex ); - if (finger.isActive()) { - - if ((emitterIndex >=0) - && (emitterIndex < NUM_FINGERS)) { - - assert(emitterIndex >=0 ); - assert(emitterIndex < NUM_FINGERS ); + if ((emitterIndex >=0) + && (emitterIndex < NUM_FINGERS)) { + _raveGloveParticleSystem.setEmitterActive(_raveGloveEmitter[emitterIndex], false); // set to false by default... + + if (finger.isActive()) { + _raveGloveParticleSystem.setEmitterActive(_raveGloveEmitter[emitterIndex], true); + if(debug) printf( "_raveGloveEmitter[%d] = %d\n", emitterIndex, _raveGloveEmitter[emitterIndex] ); glm::vec3 fingerDirection = finger.getTipPosition() - finger.getRootPosition(); @@ -317,16 +318,11 @@ void Hand::updateRaveGloveParticles(float deltaTime) { if (!_raveGloveInitialized) { - //printf( "Initializing rave glove emitters:\n" ); - //printf( "The indices of the emitters are:\n" ); - // start up the rave glove finger particles... for ( int f = 0; f< NUM_FINGERS; f ++ ) { - _raveGloveEmitter[f] = _raveGloveParticleSystem.addEmitter(); + _raveGloveEmitter[f] = _raveGloveParticleSystem.addEmitter(); assert( _raveGloveEmitter[f] >= 0 ); assert( _raveGloveEmitter[f] != NULL_EMITTER ); - - //printf( "%d\n", _raveGloveEmitter[f] ); } setRaveGloveMode(RAVE_GLOVE_EFFECTS_MODE_FIRE); @@ -339,13 +335,13 @@ void Hand::updateRaveGloveParticles(float deltaTime) { // this rave glove effect oscillates though various colors and radii that are meant to show off some effects if (_raveGloveMode == RAVE_GLOVE_EFFECTS_MODE_THROBBING_COLOR) { ParticleSystem::ParticleAttributes attributes; - float red = 0.5f + 0.5f * sinf(_raveGloveClock * 1.4f); - float green = 0.5f + 0.5f * cosf(_raveGloveClock * 1.7f); - float blue = 0.5f + 0.5f * sinf(_raveGloveClock * 2.0f); + float red = 0.5f + 0.5f * sinf(_raveGloveClock * 2.4f); + float green = 0.5f + 0.5f * cosf(_raveGloveClock * 2.7f); + float blue = 0.5f + 0.5f * sinf(_raveGloveClock * 3.0f); float alpha = 1.0f; attributes.color = glm::vec4(red, green, blue, alpha); - attributes.radius = 0.01f + 0.005f * sinf(_raveGloveClock * 2.2f); + attributes.radius = 0.01f + 0.003f * sinf(_raveGloveClock * 50.0f); attributes.modulationAmplitude = 0.0f; for ( int f = 0; f< NUM_FINGERS; f ++ ) { @@ -360,6 +356,8 @@ void Hand::updateRaveGloveParticles(float deltaTime) { } } + + void Hand::setRaveGloveMode(int mode) { _raveGloveMode = mode; @@ -376,7 +374,7 @@ void Hand::setRaveGloveMode(int mode) { if (mode == RAVE_GLOVE_EFFECTS_MODE_THROBBING_COLOR) { _raveGloveParticleSystem.setParticleRenderStyle (_raveGloveEmitter[f], PARTICLE_RENDER_STYLE_SPHERE ); _raveGloveParticleSystem.setShowingEmitterBaseParticle(_raveGloveEmitter[f], true ); - _raveGloveParticleSystem.setEmitterParticleLifespan (_raveGloveEmitter[f], 0.0f ); + _raveGloveParticleSystem.setEmitterParticleLifespan (_raveGloveEmitter[f], 0.03f ); _raveGloveParticleSystem.setEmitterThrust (_raveGloveEmitter[f], 0.0f ); _raveGloveParticleSystem.setEmitterRate (_raveGloveEmitter[f], 30.0f ); _raveGloveParticleSystem.setEmitterParticleResolution (_raveGloveEmitter[f], 20 ); @@ -650,7 +648,7 @@ void Hand::setRaveGloveMode(int mode) { _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes); //----------------------------------------- - // throb + // long sparkler //----------------------------------------- } else if (mode == RAVE_GLOVE_EFFECTS_MODE_LONG_SPARKLER) { @@ -672,6 +670,30 @@ void Hand::setRaveGloveMode(int mode) { _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_1, attributes); _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_2, attributes); _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes); + + //----------------------------------------- + // throb + //----------------------------------------- + } else if (mode == RAVE_GLOVE_EFFECTS_MODE_THROB) { + + _raveGloveParticleSystem.setParticleRenderStyle (_raveGloveEmitter[f], PARTICLE_RENDER_STYLE_SPHERE ); + _raveGloveParticleSystem.setShowingEmitterBaseParticle(_raveGloveEmitter[f], true ); + _raveGloveParticleSystem.setEmitterParticleLifespan (_raveGloveEmitter[f], 0.03 ); + _raveGloveParticleSystem.setEmitterThrust (_raveGloveEmitter[f], 0.0f ); + _raveGloveParticleSystem.setEmitterRate (_raveGloveEmitter[f], 30.0 ); + _raveGloveParticleSystem.setEmitterParticleResolution (_raveGloveEmitter[f], 20 ); + + _raveGloveParticleSystem.setParticleAttributesToDefault(&attributes); + + attributes.radius = 0.01f; + attributes.color = glm::vec4( 0.1f, 0.2f, 0.4f, 0.5f); + attributes.modulationAmplitude = 0.5; + attributes.modulationRate = 3.0; + attributes.modulationStyle = COLOR_MODULATION_STYLE_LIGHTNESS_WAVE; + _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_0, attributes); + _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_1, attributes); + _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_2, attributes); + _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes); } } } From a86ceb275961c259e3ac8f1ff03b3da059d98f62 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 31 Jul 2013 18:18:37 -0700 Subject: [PATCH 08/26] More work on full frame video. --- interface/src/Webcam.cpp | 8 +++++--- interface/src/avatar/Face.cpp | 27 +++++++++++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index b11f134fc5..3b1f1ec9ee 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -569,7 +569,8 @@ void FrameGrabber::grabFrame() { _faceColor = color; } - // convert from RGB to YV12 + // convert from RGB to YV12: see http://www.fourcc.org/yuv.php and + // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor const int ENCODED_BITS_PER_Y = 8; const int ENCODED_BITS_PER_VU = 2; const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU; @@ -627,9 +628,10 @@ void FrameGrabber::grabFrame() { // encode the frame vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); - // start the payload off with the aspect ratio + // start the payload off with the aspect ratio (zero for no face) QByteArray payload(sizeof(float), 0); - *(float*)payload.data() = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; + *(float*)payload.data() = (_videoSendMode == FACE_VIDEO) ? + (_smoothedFaceRect.size.width / _smoothedFaceRect.size.height) : 0.0f; // extract the encoded frame vpx_codec_iter_t iterator = 0; diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp index ff31241c54..72d1b5d5c2 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/Face.cpp @@ -97,7 +97,8 @@ int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { vpx_codec_iter_t iterator = 0; vpx_image_t* image; while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { - // convert from YV12 to RGB + // convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and + // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor Mat color(image->d_h, image->d_w, CV_8UC3); uchar* yline = image->planes[0]; uchar* vline = image->planes[1]; @@ -208,9 +209,19 @@ bool Face::render(float alpha) { glm::quat orientation = _owningHead->getOrientation(); glm::vec3 axis = glm::axis(orientation); glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); - float scale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + + float aspect, scale; + if (_aspectRatio == 0.0f) { + aspect = _textureSize.width / _textureSize.height; + const float FULL_FRAME_SCALE = 0.5f; + scale = FULL_FRAME_SCALE * _owningHead->getScale(); + + } else { + aspect = _aspectRatio; + scale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + } glScalef(scale, scale, scale); - + glColor4f(1.0f, 1.0f, 1.0f, alpha); Point2f points[4]; @@ -292,7 +303,7 @@ bool Face::render(float alpha) { (points[3].x - points[0].x) / _textureSize.width, (points[3].y - points[0].y) / _textureSize.height); _program->setUniformValue(_texCoordUpLocation, (points[1].x - points[0].x) / _textureSize.width, (points[1].y - points[0].y) / _textureSize.height); - _program->setUniformValue(_aspectRatioLocation, _aspectRatio); + _program->setUniformValue(_aspectRatioLocation, aspect); glEnableClientState(GL_VERTEX_ARRAY); glVertexPointer(2, GL_FLOAT, 0, 0); @@ -324,13 +335,13 @@ bool Face::render(float alpha) { glBegin(GL_QUADS); glTexCoord2f(points[0].x / _textureSize.width, points[0].y / _textureSize.height); - glVertex3f(0.5f, -0.5f / _aspectRatio, -0.5f); + glVertex3f(0.5f, -0.5f / aspect, -0.5f); glTexCoord2f(points[1].x / _textureSize.width, points[1].y / _textureSize.height); - glVertex3f(0.5f, 0.5f / _aspectRatio, -0.5f); + glVertex3f(0.5f, 0.5f / aspect, -0.5f); glTexCoord2f(points[2].x / _textureSize.width, points[2].y / _textureSize.height); - glVertex3f(-0.5f, 0.5f / _aspectRatio, -0.5f); + glVertex3f(-0.5f, 0.5f / aspect, -0.5f); glTexCoord2f(points[3].x / _textureSize.width, points[3].y / _textureSize.height); - glVertex3f(-0.5f, -0.5f / _aspectRatio, -0.5f); + glVertex3f(-0.5f, -0.5f / aspect, -0.5f); glEnd(); glDisable(GL_TEXTURE_2D); From 0cd1018e7624aaacc4dd80ecc8f85f354ecd718a Mon Sep 17 00:00:00 2001 From: Jeffrey Ventrella Date: Thu, 1 Aug 2013 10:53:20 -0700 Subject: [PATCH 09/26] some experimental work on hand rendering --- interface/src/avatar/Hand.cpp | 46 +++++++++++++++++++++++++++++++---- interface/src/avatar/Hand.h | 6 +++-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 3eec1925c0..2e409a10bf 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -14,7 +14,7 @@ #include "Util.h" #include "renderer/ProgramObject.h" -const bool SHOW_LEAP_HAND = false; +const bool SHOW_LEAP_HAND = true; using namespace std; @@ -120,8 +120,9 @@ void Hand::render(bool lookingInMirror) { glEnable(GL_RESCALE_NORMAL); if ( SHOW_LEAP_HAND ) { - renderFingerTrails(); - renderHandSpheres(); + renderLeapHands(); + //renderFingerTrails(); + //renderHandSpheres(); } } @@ -153,7 +154,42 @@ void Hand::renderRaveGloveStage() { } } -void Hand::renderHandSpheres() { + + +void Hand::renderLeapHands() { + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& hand = getPalms()[i]; + if (hand.isActive()) { + renderLeapHand(hand); + } + } +} + + +void Hand::renderLeapHand(PalmData& hand) { + + glPushMatrix(); + const float palmThickness = 0.002f; + glColor4f(0.5f, 0.5f, 0.5f, 1.0); + glm::vec3 tip = hand.getPosition(); + glm::vec3 root = hand.getPosition() + hand.getNormal() * palmThickness; + Avatar::renderJointConnectingCone(root, tip, 0.05, 0.03); + + for (size_t f = 0; f < hand.getNumFingers(); ++f) { + FingerData& finger = hand.getFingers()[f]; + if (finger.isActive()) { + glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, 0.5); + glm::vec3 tip = finger.getTipPosition(); + glm::vec3 root = finger.getRootPosition(); + Avatar::renderJointConnectingCone(root, tip, 0.001, 0.003); + } + } + + glPopMatrix(); +} + + +void Hand::renderLeapHandSpheres() { glPushMatrix(); // Draw the leap balls for (size_t i = 0; i < _leapBalls.size(); i++) { @@ -200,7 +236,7 @@ void Hand::renderHandSpheres() { glPopMatrix(); } -void Hand::renderFingerTrails() { +void Hand::renderLeapFingerTrails() { // Draw the finger root cones for (size_t i = 0; i < getNumPalms(); ++i) { PalmData& palm = getPalms()[i]; diff --git a/interface/src/avatar/Hand.h b/interface/src/avatar/Hand.h index a3a00beb96..2ee066d7b6 100755 --- a/interface/src/avatar/Hand.h +++ b/interface/src/avatar/Hand.h @@ -92,8 +92,10 @@ private: void renderRaveGloveStage(); void setRaveGloveMode(int mode); - void renderHandSpheres(); - void renderFingerTrails(); + void renderLeapHandSpheres(); + void renderLeapHands(); + void renderLeapHand(PalmData& hand); + void renderLeapFingerTrails(); void calculateGeometry(); }; From 1df7026b26b79216ea1b9b32f2af06ecb7335e81 Mon Sep 17 00:00:00 2001 From: Jeffrey Ventrella Date: Thu, 1 Aug 2013 10:54:09 -0700 Subject: [PATCH 10/26] merge? --- libraries/avatars/src/AvatarData.cpp | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 libraries/avatars/src/AvatarData.cpp diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp old mode 100755 new mode 100644 From bc1ac6b455fb131c4a1e8afbe7553c1ca083acce Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 1 Aug 2013 12:49:10 -0700 Subject: [PATCH 11/26] turn down collision sounds --- interface/src/Application.cpp | 9 ++++++--- interface/src/Application.h | 4 ---- interface/src/Util.cpp | 8 +++++++- interface/src/avatar/Avatar.cpp | 6 ------ 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index b6c2187256..8add2325d9 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -196,7 +196,6 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _isTouchPressed(false), _yawFromTouch(0.0f), _pitchFromTouch(0.0f), - _groundPlaneImpact(0.0f), _mousePressed(false), _isHoverVoxel(false), _isHoverVoxelSounding(false), @@ -2789,8 +2788,12 @@ void Application::displayOverlay() { glDisable(GL_DEPTH_TEST); glDisable(GL_LIGHTING); - // Display a single screen-size quad to - renderCollisionOverlay(_glWidget->width(), _glWidget->height(), _audio.getCollisionSoundMagnitude()); + // Display a single screen-size quad to create an alpha blended 'collision' flash + float collisionSoundMagnitude = _audio.getCollisionSoundMagnitude(); + const float VISIBLE_COLLISION_SOUND_MAGNITUDE = 0.5f; + if (collisionSoundMagnitude > VISIBLE_COLLISION_SOUND_MAGNITUDE) { + renderCollisionOverlay(_glWidget->width(), _glWidget->height(), _audio.getCollisionSoundMagnitude()); + } #ifndef _WIN32 _audio.render(_glWidget->width(), _glWidget->height()); diff --git a/interface/src/Application.h b/interface/src/Application.h index 9d3d508b72..8a4a3bf18d 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -109,8 +109,6 @@ public slots: void sendAvatarFaceVideoMessage(int frameCount, const QByteArray& data); - void setGroundPlaneImpact(float groundPlaneImpact) { _groundPlaneImpact = groundPlaneImpact; } - private slots: @@ -371,8 +369,6 @@ private: float _yawFromTouch; float _pitchFromTouch; - float _groundPlaneImpact; - VoxelDetail _mouseVoxelDragging; glm::vec3 _voxelThrust; bool _mousePressed; // true if mouse has been pressed (clear when finished) diff --git a/interface/src/Util.cpp b/interface/src/Util.cpp index 0f2c3a8955..1830a30014 100644 --- a/interface/src/Util.cpp +++ b/interface/src/Util.cpp @@ -339,6 +339,7 @@ void renderCollisionOverlay(int width, int height, float magnitude) { } void renderGroundPlaneGrid(float size, float impact) { + float IMPACT_SOUND_MAGNITUDE_FOR_RECOLOR = 0.3f; glLineWidth(2.0); glm::vec4 impactColor(1, 0, 0, 1); glm::vec3 lineColor(0.4, 0.5, 0.3); @@ -355,7 +356,12 @@ void renderGroundPlaneGrid(float size, float impact) { } // Draw the floor, colored for recent impact - glm::vec4 floorColor = impact * impactColor + (1.f - impact) * surfaceColor; + glm::vec4 floorColor; + if (impact > IMPACT_SOUND_MAGNITUDE_FOR_RECOLOR) { + floorColor = impact * impactColor + (1.f - impact) * surfaceColor; + } else { + floorColor = surfaceColor; + } glColor4fv(&floorColor.x); glBegin(GL_QUADS); glVertex3f(0, 0, 0); diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 1af82083ae..626be35d1b 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -892,21 +892,15 @@ void Avatar::updateCollisionWithSphere(glm::vec3 position, float radius, float d } void Avatar::updateCollisionWithEnvironment(float deltaTime) { - glm::vec3 up = getBodyUpDirection(); float radius = _height * 0.125f; const float ENVIRONMENT_SURFACE_ELASTICITY = 1.0f; const float ENVIRONMENT_SURFACE_DAMPING = 0.01; const float ENVIRONMENT_COLLISION_FREQUENCY = 0.05f; - const float VISIBLE_GROUND_COLLISION_VELOCITY = 0.2f; glm::vec3 penetration; if (Application::getInstance()->getEnvironment()->findCapsulePenetration( _position - up * (_pelvisFloatingHeight - radius), _position + up * (_height - _pelvisFloatingHeight - radius), radius, penetration)) { - float velocityTowardCollision = glm::dot(_velocity, glm::normalize(penetration)); - if (velocityTowardCollision > VISIBLE_GROUND_COLLISION_VELOCITY) { - Application::getInstance()->setGroundPlaneImpact(1.0f); - } _lastCollisionPosition = _position; updateCollisionSound(penetration, deltaTime, ENVIRONMENT_COLLISION_FREQUENCY); applyHardCollision(penetration, ENVIRONMENT_SURFACE_ELASTICITY, ENVIRONMENT_SURFACE_DAMPING); From ba9b5888d25543267ee050f6d310a4ad7f550ab5 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 1 Aug 2013 13:00:01 -0700 Subject: [PATCH 12/26] More work on full frame sending. --- interface/resources/shaders/face.vert | 6 +---- interface/src/Webcam.cpp | 17 +++++++----- interface/src/Webcam.h | 5 +++- interface/src/avatar/Avatar.cpp | 19 +++++++------ interface/src/avatar/Face.cpp | 39 ++++++++++++++++----------- interface/src/avatar/Face.h | 11 ++++---- 6 files changed, 57 insertions(+), 40 deletions(-) diff --git a/interface/resources/shaders/face.vert b/interface/resources/shaders/face.vert index 358a8902fe..018a85f524 100644 --- a/interface/resources/shaders/face.vert +++ b/interface/resources/shaders/face.vert @@ -17,9 +17,6 @@ uniform vec2 texCoordRight; // the texture coordinate vector from bottom to the top uniform vec2 texCoordUp; -// the aspect ratio of the image -uniform float aspectRatio; - // the depth texture uniform sampler2D depthTexture; @@ -31,6 +28,5 @@ void main(void) { const float MIN_VISIBLE_DEPTH = 1.0 / 255.0; const float MAX_VISIBLE_DEPTH = 254.0 / 255.0; gl_FrontColor = vec4(1.0, 1.0, 1.0, step(MIN_VISIBLE_DEPTH, depth) * (1.0 - step(MAX_VISIBLE_DEPTH, depth))); - gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, - (gl_Vertex.y - 0.5) / aspectRatio, depth * 2.0 - 2.0, 1.0); + gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, gl_Vertex.y - 0.5, depth - 0.5, 1.0); } diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index 3b1f1ec9ee..8340e8ba06 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -19,6 +19,7 @@ #include "Application.h" #include "Webcam.h" +#include "avatar/Face.h" using namespace cv; using namespace std; @@ -155,7 +156,7 @@ Webcam::~Webcam() { const float METERS_PER_MM = 1.0f / 1000.0f; void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midFaceDepth, - const RotatedRect& faceRect, const JointVector& joints) { + float aspectRatio, const RotatedRect& faceRect, const JointVector& joints) { IplImage colorImage = color; glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); if (_colorTextureID == 0) { @@ -192,7 +193,8 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); glBindTexture(GL_TEXTURE_2D, 0); - // store our face rect and joints, update our frame count for fps computation + // store our various data, update our frame count for fps computation + _aspectRatio = aspectRatio; _faceRect = faceRect; _joints = joints; _frameCount++; @@ -538,6 +540,10 @@ void FrameGrabber::grabFrame() { _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); } + // the aspect ratio is derived from the face rect dimensions unless we're full-frame + float aspectRatio = (_videoSendMode == FULL_FRAME_VIDEO) ? FULL_FRAME_ASPECT : + (_smoothedFaceRect.size.width / _smoothedFaceRect.size.height); + if (_videoSendMode != NO_VIDEO) { if (_colorCodec.name == 0) { // initialize encoder context(s) @@ -628,10 +634,9 @@ void FrameGrabber::grabFrame() { // encode the frame vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); - // start the payload off with the aspect ratio (zero for no face) + // start the payload off with the aspect ratio (zero for full frame) QByteArray payload(sizeof(float), 0); - *(float*)payload.data() = (_videoSendMode == FACE_VIDEO) ? - (_smoothedFaceRect.size.width / _smoothedFaceRect.size.height) : 0.0f; + *(float*)payload.data() = aspectRatio; // extract the encoded frame vpx_codec_iter_t iterator = 0; @@ -740,7 +745,7 @@ void FrameGrabber::grabFrame() { QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), - Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); + Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); } bool FrameGrabber::init() { diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index b5a26d9b13..cc5c313ece 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -50,6 +50,8 @@ public: GLuint getDepthTextureID() const { return _depthTextureID; } const cv::Size2f& getTextureSize() const { return _textureSize; } + float getAspectRatio() const { return _aspectRatio; } + const cv::RotatedRect& getFaceRect() const { return _faceRect; } const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; } @@ -63,7 +65,7 @@ public slots: void setEnabled(bool enabled); void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth, - const cv::RotatedRect& faceRect, const JointVector& joints); + float aspectRatio, const cv::RotatedRect& faceRect, const JointVector& joints); private: @@ -75,6 +77,7 @@ private: GLuint _colorTextureID; GLuint _depthTextureID; cv::Size2f _textureSize; + float _aspectRatio; cv::RotatedRect _faceRect; cv::RotatedRect _initialFaceRect; float _initialFaceDepth; diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 0bf2c94b10..bddc5d1df1 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -314,10 +314,7 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook, estimatedPosition = webcam->getEstimatedPosition(); // apply face data - _head.getFace().setColorTextureID(webcam->getColorTextureID()); - _head.getFace().setDepthTextureID(webcam->getDepthTextureID()); - _head.getFace().setTextureSize(webcam->getTextureSize()); - _head.getFace().setTextureRect(webcam->getFaceRect()); + _head.getFace().setFrameFromWebcam(); // compute and store the joint rotations const JointVector& joints = webcam->getEstimatedJoints(); @@ -334,7 +331,7 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook, } } } else { - _head.getFace().setColorTextureID(0); + _head.getFace().clearFrame(); } _head.setPitch(estimatedRotation.x * amplifyAngle.x + pitchFromTouch); _head.setYaw(estimatedRotation.y * amplifyAngle.y + yawFromTouch); @@ -1300,9 +1297,15 @@ float Avatar::getBallRenderAlpha(int ball, bool lookingInMirror) const { } void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { - - // Render the body as balls and cones - if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) { + + if (_head.getFace().isFullFrame()) { + // Render the full-frame video + float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); + if (alpha > 0.0f) { + _head.getFace().render(1.0f); + } + } else if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) { + // Render the body as balls and cones for (int b = 0; b < NUM_AVATAR_BODY_BALLS; b++) { float alpha = getBallRenderAlpha(b, lookingInMirror); diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp index 72d1b5d5c2..d982d1a4a5 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/Face.cpp @@ -17,6 +17,7 @@ #include "Avatar.h" #include "Head.h" #include "Face.h" +#include "Webcam.h" #include "renderer/ProgramObject.h" using namespace cv; @@ -25,7 +26,6 @@ ProgramObject* Face::_program = 0; int Face::_texCoordCornerLocation; int Face::_texCoordRightLocation; int Face::_texCoordUpLocation; -int Face::_aspectRatioLocation; GLuint Face::_vboID; GLuint Face::_iboID; @@ -55,11 +55,19 @@ Face::~Face() { } } -void Face::setTextureRect(const cv::RotatedRect& textureRect) { - _textureRect = textureRect; - _aspectRatio = _textureRect.size.width / _textureRect.size.height; +void Face::setFrameFromWebcam() { + Webcam* webcam = Application::getInstance()->getWebcam(); + _colorTextureID = webcam->getColorTextureID(); + _depthTextureID = webcam->getDepthTextureID(); + _textureSize = webcam->getTextureSize(); + _textureRect = webcam->getFaceRect(); + _aspectRatio = webcam->getAspectRatio(); } +void Face::clearFrame() { + _colorTextureID = 0; +} + int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { if (_colorCodec.name == 0) { // initialize decoder context @@ -210,17 +218,20 @@ bool Face::render(float alpha) { glm::vec3 axis = glm::axis(orientation); glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); - float aspect, scale; - if (_aspectRatio == 0.0f) { + float aspect, xScale, zScale; + if (_aspectRatio == FULL_FRAME_ASPECT) { aspect = _textureSize.width / _textureSize.height; const float FULL_FRAME_SCALE = 0.5f; - scale = FULL_FRAME_SCALE * _owningHead->getScale(); + xScale = FULL_FRAME_SCALE * _owningHead->getScale(); + zScale = xScale * 0.3f; } else { aspect = _aspectRatio; - scale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + xScale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + zScale = xScale * 1.5f; + glTranslatef(0.0f, -xScale * 0.75f, -xScale); } - glScalef(scale, scale, scale); + glScalef(xScale, xScale / aspect, zScale); glColor4f(1.0f, 1.0f, 1.0f, alpha); @@ -254,7 +265,6 @@ bool Face::render(float alpha) { _texCoordCornerLocation = _program->uniformLocation("texCoordCorner"); _texCoordRightLocation = _program->uniformLocation("texCoordRight"); _texCoordUpLocation = _program->uniformLocation("texCoordUp"); - _aspectRatioLocation = _program->uniformLocation("aspectRatio"); glGenBuffers(1, &_vboID); glBindBuffer(GL_ARRAY_BUFFER, _vboID); @@ -303,7 +313,6 @@ bool Face::render(float alpha) { (points[3].x - points[0].x) / _textureSize.width, (points[3].y - points[0].y) / _textureSize.height); _program->setUniformValue(_texCoordUpLocation, (points[1].x - points[0].x) / _textureSize.width, (points[1].y - points[0].y) / _textureSize.height); - _program->setUniformValue(_aspectRatioLocation, aspect); glEnableClientState(GL_VERTEX_ARRAY); glVertexPointer(2, GL_FLOAT, 0, 0); @@ -335,13 +344,13 @@ bool Face::render(float alpha) { glBegin(GL_QUADS); glTexCoord2f(points[0].x / _textureSize.width, points[0].y / _textureSize.height); - glVertex3f(0.5f, -0.5f / aspect, -0.5f); + glVertex3f(0.5f, -0.5f, 0.0f); glTexCoord2f(points[1].x / _textureSize.width, points[1].y / _textureSize.height); - glVertex3f(0.5f, 0.5f / aspect, -0.5f); + glVertex3f(0.5f, 0.5f, 0.0f); glTexCoord2f(points[2].x / _textureSize.width, points[2].y / _textureSize.height); - glVertex3f(-0.5f, 0.5f / aspect, -0.5f); + glVertex3f(-0.5f, 0.5f, 0.0f); glTexCoord2f(points[3].x / _textureSize.width, points[3].y / _textureSize.height); - glVertex3f(-0.5f, -0.5f / aspect, -0.5f); + glVertex3f(-0.5f, -0.5f, 0.0f); glEnd(); glDisable(GL_TEXTURE_2D); diff --git a/interface/src/avatar/Face.h b/interface/src/avatar/Face.h index d4812fecfb..a40111dcd3 100644 --- a/interface/src/avatar/Face.h +++ b/interface/src/avatar/Face.h @@ -20,6 +20,8 @@ class Head; class ProgramObject; +const float FULL_FRAME_ASPECT = 0.0f; + class Face : public QObject { Q_OBJECT @@ -28,10 +30,10 @@ public: Face(Head* owningHead); ~Face(); - void setColorTextureID(GLuint colorTextureID) { _colorTextureID = colorTextureID; } - void setDepthTextureID(GLuint depthTextureID) { _depthTextureID = depthTextureID; } - void setTextureSize(const cv::Size2f& textureSize) { _textureSize = textureSize; } - void setTextureRect(const cv::RotatedRect& textureRect); + bool isFullFrame() const { return _colorTextureID != 0 && _aspectRatio == FULL_FRAME_ASPECT; } + + void setFrameFromWebcam(); + void clearFrame(); int processVideoMessage(unsigned char* packetData, size_t dataBytes); @@ -68,7 +70,6 @@ private: static int _texCoordCornerLocation; static int _texCoordRightLocation; static int _texCoordUpLocation; - static int _aspectRatioLocation; static GLuint _vboID; static GLuint _iboID; }; From fcf20a7ebc9a10e2f320310f34ba067af6114a3d Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 1 Aug 2013 15:00:08 -0700 Subject: [PATCH 13/26] Sound on click, but not on hover --- interface/src/Application.cpp | 16 ++++++++++------ interface/src/Util.cpp | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8add2325d9..70a1169d90 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -855,6 +855,11 @@ void Application::mouseMoveEvent(QMouseEvent* event) { } } +const bool MAKE_SOUND_ON_VOXEL_HOVER = false; +const bool MAKE_SOUND_ON_VOXEL_CLICK = true; +const float HOVER_VOXEL_FREQUENCY = 14080.f; +const float HOVER_VOXEL_DECAY = 0.999f; + void Application::mousePressEvent(QMouseEvent* event) { if (activeWindow() == _window) { if (event->button() == Qt::LeftButton) { @@ -865,12 +870,12 @@ void Application::mousePressEvent(QMouseEvent* event) { _mouseVoxelDragging = _mouseVoxel; _mousePressed = true; maybeEditVoxelUnderCursor(); - if (_isHoverVoxel && !_isHoverVoxelSounding) { + if (MAKE_SOUND_ON_VOXEL_CLICK && _isHoverVoxel && !_isHoverVoxelSounding) { _hoverVoxelOriginalColor[0] = _hoverVoxel.red; _hoverVoxelOriginalColor[1] = _hoverVoxel.green; _hoverVoxelOriginalColor[2] = _hoverVoxel.blue; _hoverVoxelOriginalColor[3] = 1; - _audio.startCollisionSound(1.0, 14080 * _hoverVoxel.s * TREE_SCALE, 0.0, 0.999f); + _audio.startCollisionSound(1.0, HOVER_VOXEL_FREQUENCY * _hoverVoxel.s * TREE_SCALE, 0.0, HOVER_VOXEL_DECAY); _isHoverVoxelSounding = true; } @@ -2090,18 +2095,17 @@ void Application::update(float deltaTime) { if (bright < 0.01f) { hoveredNode->setColor(_hoverVoxelOriginalColor); _isHoverVoxelSounding = false; - } + } } else { // Check for a new hover voxel glm::vec4 oldVoxel(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s); _isHoverVoxel = _voxels.findRayIntersection(mouseRayOrigin, mouseRayDirection, _hoverVoxel, distance, face); - if (_isHoverVoxel && glm::vec4(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s) != oldVoxel) { - //qDebug("bing! x,y,z = %f,%f,%f\n", _hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z); + if (MAKE_SOUND_ON_VOXEL_HOVER && _isHoverVoxel && glm::vec4(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s) != oldVoxel) { _hoverVoxelOriginalColor[0] = _hoverVoxel.red; _hoverVoxelOriginalColor[1] = _hoverVoxel.green; _hoverVoxelOriginalColor[2] = _hoverVoxel.blue; _hoverVoxelOriginalColor[3] = 1; - _audio.startCollisionSound(1.0, 14080 * _hoverVoxel.s * TREE_SCALE, 0.0, 0.992f); + _audio.startCollisionSound(1.0, HOVER_VOXEL_FREQUENCY * _hoverVoxel.s * TREE_SCALE, 0.0, HOVER_VOXEL_DECAY); _isHoverVoxelSounding = true; } } diff --git a/interface/src/Util.cpp b/interface/src/Util.cpp index 1830a30014..ffb9ea25e0 100644 --- a/interface/src/Util.cpp +++ b/interface/src/Util.cpp @@ -339,7 +339,7 @@ void renderCollisionOverlay(int width, int height, float magnitude) { } void renderGroundPlaneGrid(float size, float impact) { - float IMPACT_SOUND_MAGNITUDE_FOR_RECOLOR = 0.3f; + float IMPACT_SOUND_MAGNITUDE_FOR_RECOLOR = 1.f; glLineWidth(2.0); glm::vec4 impactColor(1, 0, 0, 1); glm::vec3 lineColor(0.4, 0.5, 0.3); From 6a6c51f80ec919cc84e66a00559e1bc04f746ca5 Mon Sep 17 00:00:00 2001 From: Mark Peng Date: Thu, 1 Aug 2013 15:04:58 -0700 Subject: [PATCH 14/26] Add skeleton tracking toggle in options menu in interface. --- interface/src/Application.cpp | 1 + interface/src/Webcam.cpp | 4 ++-- interface/src/Webcam.h | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7fc9bd7ac8..0de45b3dcf 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1728,6 +1728,7 @@ void Application::initMenu() { _testPing->setChecked(true); (_fullScreenMode = optionsMenu->addAction("Fullscreen", this, SLOT(setFullscreen(bool)), Qt::Key_F))->setCheckable(true); optionsMenu->addAction("Webcam", &_webcam, SLOT(setEnabled(bool)))->setCheckable(true); + optionsMenu->addAction("Toggle Skeleton Tracking", &_webcam, SLOT(toggleSkeletonTracking(bool)))->setCheckable(true); optionsMenu->addAction("Go Home", this, SLOT(goHome())); QMenu* renderMenu = menuBar->addMenu("Render"); diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index d4fa015ba0..921f4f427b 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -32,7 +32,7 @@ int jointVectorMetaType = qRegisterMetaType("JointVector"); int matMetaType = qRegisterMetaType("cv::Mat"); int rotatedRectMetaType = qRegisterMetaType("cv::RotatedRect"); -Webcam::Webcam() : _enabled(false), _active(false), _colorTextureID(0), _depthTextureID(0) { +Webcam::Webcam() : _enabled(false), _active(false), _colorTextureID(0), _depthTextureID(0), _skeletonTrackingOn(false) { // the grabber simply runs as fast as possible _grabber = new FrameGrabber(); _grabber->moveToThread(&_grabberThread); @@ -194,7 +194,7 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean // store our face rect and joints, update our frame count for fps computation _faceRect = faceRect; - _joints = joints; + if (_skeletonTrackingOn) _joints = joints; _frameCount++; const int MAX_FPS = 60; diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index 3910bb4a19..7e27fba189 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -45,7 +45,7 @@ public: ~Webcam(); bool isActive() const { return _active; } - + GLuint getColorTextureID() const { return _colorTextureID; } GLuint getDepthTextureID() const { return _depthTextureID; } const cv::Size2f& getTextureSize() const { return _textureSize; } @@ -57,13 +57,14 @@ public: const JointVector& getEstimatedJoints() const { return _estimatedJoints; } void reset(); - void renderPreview(int screenWidth, int screenHeight); + void renderPreview(int screenWidth, int screenHeight); public slots: void setEnabled(bool enabled); void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float meanFaceDepth, const cv::RotatedRect& faceRect, const JointVector& joints); + void toggleSkeletonTracking(bool toggle) { _skeletonTrackingOn = toggle; }; private: @@ -88,6 +89,8 @@ private: glm::vec3 _estimatedPosition; glm::vec3 _estimatedRotation; JointVector _estimatedJoints; + + bool _skeletonTrackingOn; }; class FrameGrabber : public QObject { From 269e040263e2c7026fd473aa12faeeca10445d63 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 1 Aug 2013 15:27:55 -0700 Subject: [PATCH 15/26] Qt5 find package cleanup --- libraries/avatars/CMakeLists.txt | 4 ---- libraries/shared/CMakeLists.txt | 4 ---- libraries/voxels/CMakeLists.txt | 4 ++++ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/libraries/avatars/CMakeLists.txt b/libraries/avatars/CMakeLists.txt index 7968b5a003..207057e244 100644 --- a/libraries/avatars/CMakeLists.txt +++ b/libraries/avatars/CMakeLists.txt @@ -8,13 +8,9 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cm set(TARGET_NAME avatars) -find_package(Qt5Core) - include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) -qt5_use_modules(${TARGET_NAME} Core) - include(${MACRO_DIR}/IncludeGLM.cmake) include_glm(${TARGET_NAME} ${ROOT_DIR}) diff --git a/libraries/shared/CMakeLists.txt b/libraries/shared/CMakeLists.txt index b0c2771f48..cf1c603b7d 100644 --- a/libraries/shared/CMakeLists.txt +++ b/libraries/shared/CMakeLists.txt @@ -6,13 +6,9 @@ set(MACRO_DIR ${ROOT_DIR}/cmake/macros) set(TARGET_NAME shared) project(${TARGET_NAME}) -find_package(Qt5Core REQUIRED) - include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) -qt5_use_modules(${TARGET_NAME} Core) - set(EXTERNAL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external) if (WIN32) diff --git a/libraries/voxels/CMakeLists.txt b/libraries/voxels/CMakeLists.txt index 861001ed35..b6f7ae9718 100644 --- a/libraries/voxels/CMakeLists.txt +++ b/libraries/voxels/CMakeLists.txt @@ -8,9 +8,13 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cm set(TARGET_NAME voxels) +find_package(Qt5Widgets REQUIRED) + include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) +qt5_use_modules(${TARGET_NAME} Widgets) + include(${MACRO_DIR}/IncludeGLM.cmake) include_glm(${TARGET_NAME} ${ROOT_DIR}) From 9defa6255aa8b39c7cccab1d538577217a490650 Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 1 Aug 2013 15:31:09 -0700 Subject: [PATCH 16/26] disable mouse wheel voxel scale change --- interface/src/Application.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 551e666338..a82b9bc8f3 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -932,8 +932,10 @@ void Application::touchEndEvent(QTouchEvent* event) { _isTouchPressed = false; } +const bool USE_MOUSEWHEEL = false; void Application::wheelEvent(QWheelEvent* event) { - if (activeWindow() == _window) { + // Wheel Events disabled for now because they are also activated by touch look pitch up/down. + if (USE_MOUSEWHEEL && (activeWindow() == _window)) { if (checkedVoxelModeAction() == 0) { event->ignore(); return; From 12f42358d58df85b63ab54ac5a22f850c093e870 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 1 Aug 2013 15:39:53 -0700 Subject: [PATCH 17/26] Allow cycling between video send modes. --- interface/src/Application.cpp | 8 +- interface/src/Webcam.cpp | 192 ++++++++++++----------- interface/src/Webcam.h | 7 +- interface/src/avatar/Face.cpp | 285 ++++++++++++++++++++-------------- interface/src/avatar/Face.h | 3 + 5 files changed, 284 insertions(+), 211 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f331ee9407..ad40edd0db 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -960,14 +960,15 @@ void Application::sendAvatarFaceVideoMessage(int frameCount, const QByteArray& d int headerSize = packetPosition - packet; - // break the data up into submessages of the maximum size + // break the data up into submessages of the maximum size (at least one, for zero-length packets) *offsetPosition = 0; - while (*offsetPosition < data.size()) { + do { int payloadSize = min(data.size() - (int)*offsetPosition, MAX_PACKET_SIZE - headerSize); memcpy(packetPosition, data.constData() + *offsetPosition, payloadSize); getInstance()->controlledBroadcastToNodes(packet, headerSize + payloadSize, &NODE_TYPE_AVATAR_MIXER, 1); *offsetPosition += payloadSize; - } + + } while (*offsetPosition < data.size()); } // Every second, check the frame rates and other stuff @@ -1723,6 +1724,7 @@ void Application::initMenu() { _testPing->setChecked(true); (_fullScreenMode = optionsMenu->addAction("Fullscreen", this, SLOT(setFullscreen(bool)), Qt::Key_F))->setCheckable(true); optionsMenu->addAction("Webcam", &_webcam, SLOT(setEnabled(bool)))->setCheckable(true); + optionsMenu->addAction("Cycle Webcam Send Mode", _webcam.getGrabber(), SLOT(cycleVideoSendMode())); optionsMenu->addAction("Go Home", this, SLOT(goHome())); QMenu* renderMenu = menuBar->addMenu("Render"); diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index 8340e8ba06..b30d72aec6 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -156,7 +156,7 @@ Webcam::~Webcam() { const float METERS_PER_MM = 1.0f / 1000.0f; void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midFaceDepth, - float aspectRatio, const RotatedRect& faceRect, const JointVector& joints) { + float aspectRatio, const RotatedRect& faceRect, bool sending, const JointVector& joints) { IplImage colorImage = color; glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); if (_colorTextureID == 0) { @@ -196,6 +196,7 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF // store our various data, update our frame count for fps computation _aspectRatio = aspectRatio; _faceRect = faceRect; + _sending = sending; _joints = joints; _frameCount++; @@ -369,6 +370,8 @@ static void XN_CALLBACK_TYPE calibrationCompleted(SkeletonCapability& capability void FrameGrabber::cycleVideoSendMode() { _videoSendMode = (VideoSendMode)((_videoSendMode + 1) % VIDEO_SEND_MODE_COUNT); _searchWindow = cv::Rect(0, 0, 0, 0); + + destroyCodecs(); } void FrameGrabber::reset() { @@ -386,14 +389,7 @@ void FrameGrabber::shutdown() { cvReleaseCapture(&_capture); _capture = 0; } - if (_colorCodec.name != 0) { - vpx_codec_destroy(&_colorCodec); - _colorCodec.name = 0; - } - if (_depthCodec.name != 0) { - vpx_codec_destroy(&_depthCodec); - _depthCodec.name = 0; - } + destroyCodecs(); _initialized = false; thread()->quit(); @@ -469,16 +465,17 @@ void FrameGrabber::grabFrame() { color = image; } - RotatedRect faceRect; int encodedWidth; int encodedHeight; int depthBitrateMultiplier = 1; + Mat faceTransform; + float aspectRatio; if (_videoSendMode == FULL_FRAME_VIDEO) { // no need to find the face if we're sending full frame video - faceRect.center = Point2f(color.cols / 2.0f, color.rows / 2.0f); - faceRect.size = Size2f(color.cols, color.rows); + _smoothedFaceRect = RotatedRect(Point2f(color.cols / 2.0f, color.rows / 2.0f), Size2f(color.cols, color.rows), 0.0f); encodedWidth = color.cols; encodedHeight = color.rows; + aspectRatio = FULL_FRAME_ASPECT; } else { // if we don't have a search window (yet), try using the face cascade @@ -501,6 +498,7 @@ void FrameGrabber::grabFrame() { _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max); } } + RotatedRect faceRect; if (_searchWindow.area() > 0) { updateHSVFrame(color, format); @@ -517,33 +515,81 @@ void FrameGrabber::grabFrame() { encodedWidth = ENCODED_FACE_WIDTH; encodedHeight = ENCODED_FACE_HEIGHT; depthBitrateMultiplier = 2; + + // correct for 180 degree rotations + if (faceRect.angle < -90.0f) { + faceRect.angle += 180.0f; + + } else if (faceRect.angle > 90.0f) { + faceRect.angle -= 180.0f; + } + + // compute the smoothed face rect + if (_smoothedFaceRect.size.area() == 0) { + _smoothedFaceRect = faceRect; + + } else { + const float FACE_RECT_SMOOTHING = 0.9f; + _smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING); + _smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING); + _smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING); + _smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING); + _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); + } + + // use the face rect to compute the face transform, aspect ratio + Point2f sourcePoints[4]; + _smoothedFaceRect.points(sourcePoints); + Point2f destPoints[] = { Point2f(0, encodedHeight), Point2f(0, 0), Point2f(encodedWidth, 0) }; + faceTransform = getAffineTransform(sourcePoints, destPoints); + aspectRatio = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; + } + + const ushort ELEVEN_BIT_MINIMUM = 0; + const uchar EIGHT_BIT_MIDPOINT = 128; + double depthOffset; + if (!depth.empty()) { + if (_videoSendMode == FACE_VIDEO) { + // warp the face depth without interpolation (because it will contain invalid zero values) + _faceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + warpAffine(depth, _faceDepth, faceTransform, _faceDepth.size(), INTER_NEAREST); + + } else { + _faceDepth = depth; + } + _smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + + // smooth the depth over time + const ushort ELEVEN_BIT_MAXIMUM = 2047; + const float DEPTH_SMOOTHING = 0.25f; + ushort* src = _faceDepth.ptr(); + ushort* dest = _smoothedFaceDepth.ptr(); + ushort minimumDepth = numeric_limits::max(); + for (int i = 0; i < encodedHeight; i++) { + for (int j = 0; j < encodedWidth; j++) { + ushort depth = *src++; + if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { + minimumDepth = min(minimumDepth, depth); + *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); + } + dest++; + } + } + const ushort MINIMUM_DEPTH_OFFSET = 64; + const float FIXED_MID_DEPTH = 640.0f; + float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH; + + // smooth the mid face depth over time + const float MID_FACE_DEPTH_SMOOTHING = 0.5f; + _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : + glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); + + // convert from 11 to 8 bits for preview/local display + depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; + depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); } - // correct for 180 degree rotations - if (faceRect.angle < -90.0f) { - faceRect.angle += 180.0f; - - } else if (faceRect.angle > 90.0f) { - faceRect.angle -= 180.0f; - } - - // compute the smoothed face rect - if (_smoothedFaceRect.size.area() == 0) { - _smoothedFaceRect = faceRect; - - } else { - const float FACE_RECT_SMOOTHING = 0.9f; - _smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING); - _smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING); - _smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING); - _smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING); - _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); - } - - // the aspect ratio is derived from the face rect dimensions unless we're full-frame - float aspectRatio = (_videoSendMode == FULL_FRAME_VIDEO) ? FULL_FRAME_ASPECT : - (_smoothedFaceRect.size.width / _smoothedFaceRect.size.height); - + QByteArray payload; if (_videoSendMode != NO_VIDEO) { if (_colorCodec.name == 0) { // initialize encoder context(s) @@ -565,11 +611,7 @@ void FrameGrabber::grabFrame() { if (_videoSendMode == FACE_VIDEO) { // resize/rotate face into encoding rectangle _faceColor.create(encodedHeight, encodedWidth, CV_8UC3); - Point2f sourcePoints[4]; - _smoothedFaceRect.points(sourcePoints); - Point2f destPoints[] = { Point2f(0, encodedHeight), Point2f(0, 0), Point2f(encodedWidth, 0) }; - transform = getAffineTransform(sourcePoints, destPoints); - warpAffine(color, _faceColor, transform, _faceColor.size()); + warpAffine(color, _faceColor, faceTransform, _faceColor.size()); } else { _faceColor = color; @@ -635,8 +677,7 @@ void FrameGrabber::grabFrame() { vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); // start the payload off with the aspect ratio (zero for full frame) - QByteArray payload(sizeof(float), 0); - *(float*)payload.data() = aspectRatio; + payload.append((const char*)&aspectRatio, sizeof(float)); // extract the encoded frame vpx_codec_iter_t iterator = 0; @@ -650,48 +691,7 @@ void FrameGrabber::grabFrame() { } if (!depth.empty()) { - if (_videoSendMode == FACE_VIDEO) { - // warp the face depth without interpolation (because it will contain invalid zero values) - _faceDepth.create(encodedHeight, encodedWidth, CV_16UC1); - warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST); - - } else { - _faceDepth = depth; - } - _smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1); - - // smooth the depth over time - const ushort ELEVEN_BIT_MINIMUM = 0; - const ushort ELEVEN_BIT_MAXIMUM = 2047; - const float DEPTH_SMOOTHING = 0.25f; - ushort* src = _faceDepth.ptr(); - ushort* dest = _smoothedFaceDepth.ptr(); - ushort minimumDepth = numeric_limits::max(); - for (int i = 0; i < encodedHeight; i++) { - for (int j = 0; j < encodedWidth; j++) { - ushort depth = *src++; - if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { - minimumDepth = min(minimumDepth, depth); - *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); - } - dest++; - } - } - const ushort MINIMUM_DEPTH_OFFSET = 64; - const float FIXED_MID_DEPTH = 640.0f; - float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH; - - // smooth the mid face depth over time - const float MID_FACE_DEPTH_SMOOTHING = 0.5f; - _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : - glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); - - // convert from 11 to 8 bits for preview/local display - const uchar EIGHT_BIT_MIDPOINT = 128; - double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; - depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); - - // likewise for the encoded representation + // convert with mask uchar* yline = vpxImage.planes[0]; uchar* vline = vpxImage.planes[1]; uchar* uline = vpxImage.planes[2]; @@ -738,14 +738,15 @@ void FrameGrabber::grabFrame() { } } } - - QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", - Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); } + QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", + Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); + QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), - Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); + Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(bool, !payload.isEmpty()), + Q_ARG(JointVector, joints)); } bool FrameGrabber::init() { @@ -814,6 +815,17 @@ void FrameGrabber::updateHSVFrame(const Mat& frame, int format) { inRange(_hsvFrame, Scalar(0, 55, 65), Scalar(180, 256, 256), _mask); } +void FrameGrabber::destroyCodecs() { + if (_colorCodec.name != 0) { + vpx_codec_destroy(&_colorCodec); + _colorCodec.name = 0; + } + if (_depthCodec.name != 0) { + vpx_codec_destroy(&_depthCodec); + _depthCodec.name = 0; + } +} + Joint::Joint(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& projected) : isValid(true), position(position), rotation(rotation), projected(projected) { } diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index cc5c313ece..12ef2558db 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -44,7 +44,10 @@ public: Webcam(); ~Webcam(); + FrameGrabber* getGrabber() { return _grabber; } + bool isActive() const { return _active; } + bool isSending() const { return _sending; } GLuint getColorTextureID() const { return _colorTextureID; } GLuint getDepthTextureID() const { return _depthTextureID; } @@ -65,7 +68,7 @@ public slots: void setEnabled(bool enabled); void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth, - float aspectRatio, const cv::RotatedRect& faceRect, const JointVector& joints); + float aspectRatio, const cv::RotatedRect& faceRect, bool sending, const JointVector& joints); private: @@ -74,6 +77,7 @@ private: bool _enabled; bool _active; + bool _sending; GLuint _colorTextureID; GLuint _depthTextureID; cv::Size2f _textureSize; @@ -114,6 +118,7 @@ private: bool init(); void updateHSVFrame(const cv::Mat& frame, int format); + void destroyCodecs(); bool _initialized; VideoSendMode _videoSendMode; diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp index d982d1a4a5..0541b3c21c 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/Face.cpp @@ -57,11 +57,16 @@ Face::~Face() { void Face::setFrameFromWebcam() { Webcam* webcam = Application::getInstance()->getWebcam(); - _colorTextureID = webcam->getColorTextureID(); - _depthTextureID = webcam->getDepthTextureID(); - _textureSize = webcam->getTextureSize(); - _textureRect = webcam->getFaceRect(); - _aspectRatio = webcam->getAspectRatio(); + if (webcam->isSending()) { + _colorTextureID = webcam->getColorTextureID(); + _depthTextureID = webcam->getDepthTextureID(); + _textureSize = webcam->getTextureSize(); + _textureRect = webcam->getFaceRect(); + _aspectRatio = webcam->getAspectRatio(); + + } else { + clearFrame(); + } } void Face::clearFrame() { @@ -69,11 +74,6 @@ void Face::clearFrame() { } int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { - if (_colorCodec.name == 0) { - // initialize decoder context - vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0); - } - // skip the header unsigned char* packetPosition = packetData; int frameCount = *(uint32_t*)packetPosition; @@ -97,111 +97,135 @@ int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { int payloadSize = dataBytes - (packetPosition - packetData); memcpy(_arrivingFrame.data() + frameOffset, packetPosition, payloadSize); - if ((_frameBytesRemaining -= payloadSize) <= 0) { - float aspectRatio = *(const float*)_arrivingFrame.constData(); - size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float)); - const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t)); - vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0); - vpx_codec_iter_t iterator = 0; - vpx_image_t* image; - while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { - // convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and - // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor - Mat color(image->d_h, image->d_w, CV_8UC3); - uchar* yline = image->planes[0]; - uchar* vline = image->planes[1]; - uchar* uline = image->planes[2]; - const int RED_V_WEIGHT = (int)(1.403 * 256); - const int GREEN_V_WEIGHT = (int)(0.714 * 256); - const int GREEN_U_WEIGHT = (int)(0.344 * 256); - const int BLUE_U_WEIGHT = (int)(1.773 * 256); - for (int i = 0; i < image->d_h; i += 2) { - uchar* ysrc = yline; - uchar* vsrc = vline; - uchar* usrc = uline; - for (int j = 0; j < image->d_w; j += 2) { - uchar* tl = color.ptr(i, j); - uchar* tr = color.ptr(i, j + 1); - uchar* bl = color.ptr(i + 1, j); - uchar* br = color.ptr(i + 1, j + 1); - - int v = *vsrc++ - 128; - int u = *usrc++ - 128; - - int redOffset = (RED_V_WEIGHT * v) >> 8; - int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; - int blueOffset = (BLUE_U_WEIGHT * u) >> 8; - - int ytl = ysrc[0]; - int ytr = ysrc[1]; - int ybl = ysrc[image->w]; - int ybr = ysrc[image->w + 1]; - ysrc += 2; - - tl[0] = ytl + redOffset; - tl[1] = ytl - greenOffset; - tl[2] = ytl + blueOffset; - - tr[0] = ytr + redOffset; - tr[1] = ytr - greenOffset; - tr[2] = ytr + blueOffset; - - bl[0] = ybl + redOffset; - bl[1] = ybl - greenOffset; - bl[2] = ybl + blueOffset; - - br[0] = ybr + redOffset; - br[1] = ybr - greenOffset; - br[2] = ybr + blueOffset; - } - yline += image->stride[0] * 2; - vline += image->stride[1]; - uline += image->stride[2]; + if ((_frameBytesRemaining -= payloadSize) > 0) { + return dataBytes; // wait for the rest of the frame + } + + if (frameSize == 0) { + // destroy the codecs, if we have any + destroyCodecs(); + + // disables video data + QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, Mat()), + Q_ARG(cv::Mat, Mat()), Q_ARG(float, 0.0f)); + return dataBytes; + } + + // the switch from full frame to not (or vice versa) requires us to reinit the codecs + float aspectRatio = *(const float*)_arrivingFrame.constData(); + bool fullFrame = (aspectRatio == FULL_FRAME_ASPECT); + if (fullFrame != _lastFullFrame) { + destroyCodecs(); + _lastFullFrame = fullFrame; + } + + if (_colorCodec.name == 0) { + // initialize decoder context + vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0); + } + + size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float)); + const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t)); + vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0); + vpx_codec_iter_t iterator = 0; + vpx_image_t* image; + while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { + // convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and + // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor + Mat color(image->d_h, image->d_w, CV_8UC3); + uchar* yline = image->planes[0]; + uchar* vline = image->planes[1]; + uchar* uline = image->planes[2]; + const int RED_V_WEIGHT = (int)(1.403 * 256); + const int GREEN_V_WEIGHT = (int)(0.714 * 256); + const int GREEN_U_WEIGHT = (int)(0.344 * 256); + const int BLUE_U_WEIGHT = (int)(1.773 * 256); + for (int i = 0; i < image->d_h; i += 2) { + uchar* ysrc = yline; + uchar* vsrc = vline; + uchar* usrc = uline; + for (int j = 0; j < image->d_w; j += 2) { + uchar* tl = color.ptr(i, j); + uchar* tr = color.ptr(i, j + 1); + uchar* bl = color.ptr(i + 1, j); + uchar* br = color.ptr(i + 1, j + 1); + + int v = *vsrc++ - 128; + int u = *usrc++ - 128; + + int redOffset = (RED_V_WEIGHT * v) >> 8; + int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; + int blueOffset = (BLUE_U_WEIGHT * u) >> 8; + + int ytl = ysrc[0]; + int ytr = ysrc[1]; + int ybl = ysrc[image->w]; + int ybr = ysrc[image->w + 1]; + ysrc += 2; + + tl[0] = ytl + redOffset; + tl[1] = ytl - greenOffset; + tl[2] = ytl + blueOffset; + + tr[0] = ytr + redOffset; + tr[1] = ytr - greenOffset; + tr[2] = ytr + blueOffset; + + bl[0] = ybl + redOffset; + bl[1] = ybl - greenOffset; + bl[2] = ybl + blueOffset; + + br[0] = ybr + redOffset; + br[1] = ybr - greenOffset; + br[2] = ybr + blueOffset; } - Mat depth; - - const uint8_t* depthData = colorData + colorSize; - int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData()); - if (depthSize > 0) { - if (_depthCodec.name == 0) { - // initialize decoder context - vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0); - } - vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0); - vpx_codec_iter_t iterator = 0; - vpx_image_t* image; - while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) { - depth.create(image->d_h, image->d_w, CV_8UC1); - uchar* yline = image->planes[0]; - uchar* vline = image->planes[1]; - const uchar EIGHT_BIT_MAXIMUM = 255; - const uchar MASK_THRESHOLD = 192; - for (int i = 0; i < image->d_h; i += 2) { - uchar* ysrc = yline; - uchar* vsrc = vline; - for (int j = 0; j < image->d_w; j += 2) { - if (*vsrc++ < MASK_THRESHOLD) { - *depth.ptr(i, j) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM; - - } else { - *depth.ptr(i, j) = ysrc[0]; - *depth.ptr(i, j + 1) = ysrc[1]; - *depth.ptr(i + 1, j) = ysrc[image->stride[0]]; - *depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1]; - } - ysrc += 2; - } - yline += image->stride[0] * 2; - vline += image->stride[1]; - } - } - } - QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color), - Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio)); + yline += image->stride[0] * 2; + vline += image->stride[1]; + uline += image->stride[2]; } + Mat depth; + + const uint8_t* depthData = colorData + colorSize; + int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData()); + if (depthSize > 0) { + if (_depthCodec.name == 0) { + // initialize decoder context + vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0); + } + vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0); + vpx_codec_iter_t iterator = 0; + vpx_image_t* image; + while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) { + depth.create(image->d_h, image->d_w, CV_8UC1); + uchar* yline = image->planes[0]; + uchar* vline = image->planes[1]; + const uchar EIGHT_BIT_MAXIMUM = 255; + const uchar MASK_THRESHOLD = 192; + for (int i = 0; i < image->d_h; i += 2) { + uchar* ysrc = yline; + uchar* vsrc = vline; + for (int j = 0; j < image->d_w; j += 2) { + if (*vsrc++ < MASK_THRESHOLD) { + *depth.ptr(i, j) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM; + + } else { + *depth.ptr(i, j) = ysrc[0]; + *depth.ptr(i, j + 1) = ysrc[1]; + *depth.ptr(i + 1, j) = ysrc[image->stride[0]]; + *depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1]; + } + ysrc += 2; + } + yline += image->stride[0] * 2; + vline += image->stride[1]; + } + } + } + QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color), + Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio)); } return dataBytes; @@ -368,23 +392,40 @@ void Face::cycleRenderMode() { } void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) { + if (color.empty()) { + // release our textures, if any; there's no more video + if (_colorTextureID != 0) { + glDeleteTextures(1, &_colorTextureID); + _colorTextureID = 0; + } + if (_depthTextureID != 0) { + glDeleteTextures(1, &_depthTextureID); + _depthTextureID = 0; + } + return; + } + if (_colorTextureID == 0) { glGenTextures(1, &_colorTextureID); - glBindTexture(GL_TEXTURE_2D, _colorTextureID); + } + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + bool recreateTextures = (_textureSize.width != color.cols || _textureSize.height != color.rows); + if (recreateTextures) { glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, color.cols, color.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); _textureSize = color.size(); _textureRect = RotatedRect(Point2f(color.cols * 0.5f, color.rows * 0.5f), _textureSize, 0.0f); - + } else { - glBindTexture(GL_TEXTURE_2D, _colorTextureID); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, color.cols, color.rows, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); } if (!depth.empty()) { if (_depthTextureID == 0) { glGenTextures(1, &_depthTextureID); - glBindTexture(GL_TEXTURE_2D, _depthTextureID); + } + glBindTexture(GL_TEXTURE_2D, _depthTextureID); + if (recreateTextures) { glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depth.cols, depth.rows, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); @@ -400,3 +441,13 @@ void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRati _aspectRatio = aspectRatio; } +void Face::destroyCodecs() { + if (_colorCodec.name != 0) { + vpx_codec_destroy(&_colorCodec); + _colorCodec.name = 0; + } + if (_depthCodec.name != 0) { + vpx_codec_destroy(&_depthCodec); + _depthCodec.name = 0; + } +} diff --git a/interface/src/avatar/Face.h b/interface/src/avatar/Face.h index a40111dcd3..893318f186 100644 --- a/interface/src/avatar/Face.h +++ b/interface/src/avatar/Face.h @@ -51,6 +51,8 @@ private: enum RenderMode { MESH, POINTS, RENDER_MODE_COUNT }; + void destroyCodecs(); + Head* _owningHead; RenderMode _renderMode; GLuint _colorTextureID; @@ -61,6 +63,7 @@ private: vpx_codec_ctx_t _colorCodec; vpx_codec_ctx_t _depthCodec; + bool _lastFullFrame; QByteArray _arrivingFrame; int _frameCount; From a0ba78867906748b1ae382c9fa3968aa4c17b7f4 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 1 Aug 2013 15:40:39 -0700 Subject: [PATCH 18/26] implement grid landscape import --- interface/src/Application.cpp | 180 +++++++++++++++++--------- libraries/voxels/src/VoxelConstants.h | 2 + voxel-server/src/main.cpp | 15 +++ 3 files changed, 136 insertions(+), 61 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f331ee9407..87f52ef285 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1490,16 +1490,18 @@ bool Application::sendVoxelsOperation(VoxelNode* node, void* extraData) { uint64_t now = usecTimestampNow(); // dynamically sleep until we need to fire off the next set of voxels - const uint64_t CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS = 1000 * 5; // 1 packet every 10 milliseconds uint64_t elapsed = now - args->lastSendTime; int usecToSleep = CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS - elapsed; if (usecToSleep > 0) { - qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", - args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed, usecToSleep); + //qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", + // args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed, usecToSleep); + + Application::getInstance()->timer(); + usleep(usecToSleep); } else { - qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", - args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed); + //qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", + // args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed); } args->lastSendTime = now; } @@ -1567,70 +1569,127 @@ void Application::importVoxelsToClipboard() { void Application::importVoxels() { QString desktopLocation = QDesktopServices::storageLocation(QDesktopServices::DesktopLocation); - QString fileNameString = QFileDialog::getOpenFileName(_glWidget, tr("Import Voxels"), desktopLocation, + + QStringList fileNameStringList = QFileDialog::getOpenFileNames(_glWidget, tr("Import Voxels"), desktopLocation, tr(IMPORT_FILE_TYPES)); - QByteArray fileNameAscii = fileNameString.toAscii(); - const char* fileName = fileNameAscii.data(); + + // remember the "selected" voxel point before we do any importing... + float originalX = _mouseVoxel.x; + float originalZ = _mouseVoxel.z; + + const int PNG_TYPE_NAME_LENGTH = 4; + const int SVO_TYPE_NAME_LENGTH = 4; + const int SCH_TYPE_NAME_LENGTH = 10; + + for (int i = 0; i < fileNameStringList.size(); i++) { + QString fileNameString = fileNameStringList.at(i); + QByteArray fileNameAscii = fileNameString.toLocal8Bit(); + const char* fileName = fileNameAscii.data(); - VoxelTree importVoxels; - if (fileNameString.endsWith(".png", Qt::CaseInsensitive)) { - QImage pngImage = QImage(fileName); - if (pngImage.height() != pngImage.width()) { - qDebug("ERROR: Bad PNG size: height != width.\n"); - return; + int fileTypeNameLength = 0; + VoxelTree importVoxels; + if (fileNameString.endsWith(".png", Qt::CaseInsensitive)) { + QImage pngImage = QImage(fileName); + fileTypeNameLength = PNG_TYPE_NAME_LENGTH; + if (pngImage.height() != pngImage.width()) { + qDebug("ERROR: Bad PNG size: height != width.\n"); + return; + } + + const uint32_t* pixels; + if (pngImage.format() == QImage::Format_ARGB32) { + pixels = reinterpret_cast(pngImage.constBits()); + } else { + QImage tmp = pngImage.convertToFormat(QImage::Format_ARGB32); + pixels = reinterpret_cast(tmp.constBits()); + } + + importVoxels.readFromSquareARGB32Pixels(pixels, pngImage.height()); + } else if (fileNameString.endsWith(".svo", Qt::CaseInsensitive)) { + importVoxels.readFromSVOFile(fileName); + fileTypeNameLength = SVO_TYPE_NAME_LENGTH; + } else if (fileNameString.endsWith(".schematic", Qt::CaseInsensitive)) { + importVoxels.readFromSchematicFile(fileName); + fileTypeNameLength = SCH_TYPE_NAME_LENGTH; } - const uint32_t* pixels; - if (pngImage.format() == QImage::Format_ARGB32) { - pixels = reinterpret_cast(pngImage.constBits()); + int indexOfFirstPeriod = fileNameString.indexOf('.'); + + QString fileCoord = fileNameString.mid(indexOfFirstPeriod + 1, + fileNameString.length() - indexOfFirstPeriod - fileTypeNameLength - 1); + + indexOfFirstPeriod = fileCoord.indexOf('.'); + QString columnNumString = fileCoord.right(fileCoord.length() - indexOfFirstPeriod - 1); + QString rowNumString = fileCoord.left(indexOfFirstPeriod); + + int columnNum = columnNumString.toFloat(); + int rowNum = rowNumString.toFloat(); + + qDebug("columnNum: %d\t rowNum: %d\n", columnNum, rowNum); + + _mouseVoxel.x = originalX + (columnNum - 1) * _mouseVoxel.s; + _mouseVoxel.z = originalZ + (rowNum - 1) * _mouseVoxel.s; + + VoxelNode* selectedNode = _voxels.getVoxelAt(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); + + // Recurse the Import Voxels tree, where everything is root relative, and send all the colored voxels to + // the server as an set voxel message, this will also rebase the voxels to the new location + unsigned char* calculatedOctCode = NULL; + SendVoxelsOperationArgs args; + args.lastSendTime = usecTimestampNow(); + args.packetsSent = 0; + args.bytesSent = 0; + + int numBytesPacketHeader = populateTypeAndVersion(args.messageBuffer, PACKET_TYPE_SET_VOXEL_DESTRUCTIVE); + + unsigned short int* sequenceAt = (unsigned short int*)&args.messageBuffer[numBytesPacketHeader]; + *sequenceAt = 0; + args.bufferInUse = numBytesPacketHeader + sizeof(unsigned short int); // set to command + sequence + + // we only need the selected voxel to get the newBaseOctCode, which we can actually calculate from the + // voxel size/position details. + if (selectedNode) { + args.newBaseOctCode = selectedNode->getOctalCode(); } else { - QImage tmp = pngImage.convertToFormat(QImage::Format_ARGB32); - pixels = reinterpret_cast(tmp.constBits()); + args.newBaseOctCode = calculatedOctCode = pointToVoxel(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); + } + + qDebug("column:%d, row:%d, voxel:%f,%f,%f,%f\n", columnNum, rowNum, _mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s ); + + // send the insert/paste of these voxels + importVoxels.recurseTreeWithOperation(sendVoxelsOperation, &args); + + // If we have voxels left in the packet, then send the packet + if (args.bufferInUse > (numBytesPacketHeader + sizeof(unsigned short int))) { + controlledBroadcastToNodes(args.messageBuffer, args.bufferInUse, & NODE_TYPE_VOXEL_SERVER, 1); + + + args.packetsSent++; + args.bytesSent += args.bufferInUse; + + uint64_t now = usecTimestampNow(); + // dynamically sleep until we need to fire off the next set of voxels + uint64_t elapsed = now - args.lastSendTime; + int usecToSleep = CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS - elapsed; + + if (usecToSleep > 0) { + //qDebug("after sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", + // args.packetsSent, (long long int)args.bytesSent, (long long int)elapsed, usecToSleep); + usleep(usecToSleep); + } else { + //qDebug("after sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", + // args.packetsSent, (long long int)args.bytesSent, (long long int)elapsed); + } + args.lastSendTime = now; } - importVoxels.readFromSquareARGB32Pixels(pixels, pngImage.height()); - } else if (fileNameString.endsWith(".svo", Qt::CaseInsensitive)) { - importVoxels.readFromSVOFile(fileName); - } else if (fileNameString.endsWith(".schematic", Qt::CaseInsensitive)) { - importVoxels.readFromSchematicFile(fileName); + if (calculatedOctCode) { + delete[] calculatedOctCode; + } + } - VoxelNode* selectedNode = _voxels.getVoxelAt(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); - - // Recurse the Import Voxels tree, where everything is root relative, and send all the colored voxels to - // the server as an set voxel message, this will also rebase the voxels to the new location - unsigned char* calculatedOctCode = NULL; - SendVoxelsOperationArgs args; - args.lastSendTime = usecTimestampNow(); - args.packetsSent = 0; - args.bytesSent = 0; - - int numBytesPacketHeader = populateTypeAndVersion(args.messageBuffer, PACKET_TYPE_SET_VOXEL_DESTRUCTIVE); - - unsigned short int* sequenceAt = (unsigned short int*)&args.messageBuffer[numBytesPacketHeader]; - *sequenceAt = 0; - args.bufferInUse = numBytesPacketHeader + sizeof(unsigned short int); // set to command + sequence - - // we only need the selected voxel to get the newBaseOctCode, which we can actually calculate from the - // voxel size/position details. - if (selectedNode) { - args.newBaseOctCode = selectedNode->getOctalCode(); - } else { - args.newBaseOctCode = calculatedOctCode = pointToVoxel(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); - } - - importVoxels.recurseTreeWithOperation(sendVoxelsOperation, &args); - - // If we have voxels left in the packet, then send the packet - if (args.bufferInUse > (numBytesPacketHeader + sizeof(unsigned short int))) { - controlledBroadcastToNodes(args.messageBuffer, args.bufferInUse, & NODE_TYPE_VOXEL_SERVER, 1); - } - - if (calculatedOctCode) { - delete[] calculatedOctCode; - } - // restore the main window's active state _window->activateWindow(); } @@ -3454,6 +3513,7 @@ void* Application::networkReceive(void* args) { } if (NodeList::getInstance()->getNodeSocket()->receive(&senderAddress, app->_incomingPacket, &bytesReceived)) { + app->_packetCount++; app->_bytesCount += bytesReceived; @@ -3498,13 +3558,11 @@ void* Application::networkReceive(void* args) { Node* voxelServer = NodeList::getInstance()->soloNodeOfType(NODE_TYPE_VOXEL_SERVER); if (voxelServer && socketMatch(voxelServer->getActiveSocket(), &senderAddress)) { voxelServer->lock(); - if (messageData[0] == PACKET_TYPE_ENVIRONMENT_DATA) { app->_environment.parseData(&senderAddress, messageData, messageLength); } else { app->_voxels.parseData(messageData, messageLength); } - voxelServer->unlock(); } } diff --git a/libraries/voxels/src/VoxelConstants.h b/libraries/voxels/src/VoxelConstants.h index 55dff0c69a..4b971e9515 100644 --- a/libraries/voxels/src/VoxelConstants.h +++ b/libraries/voxels/src/VoxelConstants.h @@ -41,4 +41,6 @@ const glBufferIndex GLBUFFER_INDEX_UNKNOWN = ULONG_MAX; const float SIXTY_FPS_IN_MILLISECONDS = 1000.0f / 60.0f; const float VIEW_CULLING_RATE_IN_MILLISECONDS = 1000.0f; // once a second is fine +const uint64_t CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS = 1000 * 5; // 1 packet every 50 milliseconds + #endif \ No newline at end of file diff --git a/voxel-server/src/main.cpp b/voxel-server/src/main.cpp index c009eec2a5..d902f5dfeb 100644 --- a/voxel-server/src/main.cpp +++ b/voxel-server/src/main.cpp @@ -61,6 +61,7 @@ bool wantColorRandomizer = false; bool debugVoxelSending = false; bool shouldShowAnimationDebug = false; bool displayVoxelStats = false; +bool debugVoxelReceiving = false; EnvironmentData environmentData[3]; @@ -426,6 +427,8 @@ void attachVoxelNodeDataToNode(Node* newNode) { } } +int receivedPacketCount = 0; + int main(int argc, const char * argv[]) { pthread_mutex_init(&::treeLock, NULL); @@ -455,6 +458,10 @@ int main(int argc, const char * argv[]) { ::debugVoxelSending = cmdOptionExists(argc, argv, DEBUG_VOXEL_SENDING); printf("debugVoxelSending=%s\n", debug::valueOf(::debugVoxelSending)); + const char* DEBUG_VOXEL_RECEIVING = "--debugVoxelReceiving"; + ::debugVoxelReceiving = cmdOptionExists(argc, argv, DEBUG_VOXEL_RECEIVING); + printf("debugVoxelReceiving=%s\n", debug::valueOf(::debugVoxelReceiving)); + const char* WANT_ANIMATION_DEBUG = "--shouldShowAnimationDebug"; ::shouldShowAnimationDebug = cmdOptionExists(argc, argv, WANT_ANIMATION_DEBUG); printf("shouldShowAnimationDebug=%s\n", debug::valueOf(::shouldShowAnimationDebug)); @@ -584,12 +591,20 @@ int main(int argc, const char * argv[]) { destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", ::shouldShowAnimationDebug); + ::receivedPacketCount++; + unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader))); if (::shouldShowAnimationDebug) { printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n", destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", receivedBytes,itemNumber); } + + if (::debugVoxelReceiving) { + printf("got %s - %d command from client receivedBytes=%ld itemNumber=%d\n", + destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", + ::receivedPacketCount, receivedBytes,itemNumber); + } int atByte = numBytesPacketHeader + sizeof(itemNumber); unsigned char* voxelData = (unsigned char*)&packetData[atByte]; while (atByte < receivedBytes) { From 8f955346448d64a3d70122fda112d6968e00de7f Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 1 Aug 2013 16:04:38 -0700 Subject: [PATCH 19/26] Fix jittering in view from body slowly following head --- interface/src/Application.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index a82b9bc8f3..c297e578ca 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2324,19 +2324,17 @@ void Application::update(float deltaTime) { void Application::updateAvatar(float deltaTime) { - // When head is rotated via touch/mouse look, slowly turn body to follow - const float BODY_FOLLOW_HEAD_RATE = 0.5f; - // update body yaw by body yaw delta + // rotate body yaw for yaw received from multitouch _myAvatar.setOrientation(_myAvatar.getOrientation() - * glm::quat(glm::vec3(0, _yawFromTouch * deltaTime * BODY_FOLLOW_HEAD_RATE, 0) * deltaTime)); - _yawFromTouch -= _yawFromTouch * deltaTime * BODY_FOLLOW_HEAD_RATE; + * glm::quat(glm::vec3(0, _yawFromTouch * deltaTime, 0))); + _yawFromTouch = 0.f; // Update my avatar's state from gyros and/or webcam _myAvatar.updateFromGyrosAndOrWebcam(_gyroLook->isChecked(), glm::vec3(_headCameraPitchYawScale, _headCameraPitchYawScale, _headCameraPitchYawScale), - _yawFromTouch, + 0.f, _pitchFromTouch); if (_serialHeadSensor.isActive()) { From c038e2c9baff78fb266420b33fd612f201061aa8 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 1 Aug 2013 18:32:48 -0700 Subject: [PATCH 20/26] Wrong array index here, and finger needs to be flagged as active so that the avatar mixer will encode it. --- libraries/avatars/src/HandData.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp index 7bb6eebdc0..639b4adc0f 100755 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -83,7 +83,8 @@ void HandData::decodeRemoteData(const std::vector& fingerVectors) { palm.setRawPosition(fingerVectors[vectorIndex++]); palm.setRawNormal(fingerVectors[vectorIndex++]); for (size_t f = 0; f < NUM_FINGERS_PER_HAND; ++f) { - FingerData& finger = palm.getFingers()[i]; + FingerData& finger = palm.getFingers()[f]; + finger.setActive(true); finger.setRawTipPosition(fingerVectors[vectorIndex++]); finger.setRawRootPosition(fingerVectors[vectorIndex++]); } From 69cac690675377be51e1a7e001de2be277a98e38 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 1 Aug 2013 18:56:14 -0700 Subject: [PATCH 21/26] Go ahead and show the leap hand by default, but don't include the fingers if the palms are inactive (!) --- interface/src/avatar/Hand.cpp | 2 +- libraries/avatars/src/HandData.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 28b1af0603..f70d9aff13 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -14,7 +14,7 @@ #include "Util.h" #include "renderer/ProgramObject.h" -const bool SHOW_LEAP_HAND = false; +const bool SHOW_LEAP_HAND = true; using namespace std; diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp index 639b4adc0f..ebef2b1247 100755 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -53,6 +53,9 @@ void HandData::encodeRemoteData(std::vector& fingerVectors) { fingerVectors.clear(); for (size_t i = 0; i < getNumPalms(); ++i) { PalmData& palm = getPalms()[i]; + if (!palm.isActive()) { + continue; + } fingerVectors.push_back(palm.getRawPosition()); fingerVectors.push_back(palm.getRawNormal()); for (size_t f = 0; f < palm.getNumFingers(); ++f) { From 11664f94463a0bc4e73aa634a92a5af8930194a3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 2 Aug 2013 10:12:21 -0700 Subject: [PATCH 22/26] Have the particle system default to off. --- interface/src/Application.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f37241a720..0f87e2f063 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1816,7 +1816,6 @@ void Application::initMenu() { (_renderLookatIndicatorOn = renderMenu->addAction("Lookat Indicator"))->setCheckable(true); _renderLookatIndicatorOn->setChecked(true); (_renderParticleSystemOn = renderMenu->addAction("Particle System"))->setCheckable(true); - _renderParticleSystemOn->setChecked(true); (_manualFirstPerson = renderMenu->addAction( "First Person", this, SLOT(setRenderFirstPerson(bool)), Qt::Key_P))->setCheckable(true); (_manualThirdPerson = renderMenu->addAction( From 8fc800acffbc4689a852ff13c5326cea35216999 Mon Sep 17 00:00:00 2001 From: Jeffrey Ventrella Date: Fri, 2 Aug 2013 10:59:04 -0700 Subject: [PATCH 23/26] some work on hands --- interface/src/Application.cpp | 1 + interface/src/avatar/Hand.cpp | 10 +++++----- libraries/avatars/src/AvatarData.cpp | 17 +++++++++++++++++ libraries/avatars/src/HandData.cpp | 5 ++++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 9b292b3d50..8048d437b7 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2024,6 +2024,7 @@ void Application::renderLookatIndicator(glm::vec3 pointOfInterest, Camera& which } void Application::update(float deltaTime) { + // Use Transmitter Hand to move hand if connected, else use mouse if (_myTransmitter.isConnected()) { const float HAND_FORCE_SCALING = 0.01f; diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 2e409a10bf..7e5652c5d5 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -51,6 +51,7 @@ void Hand::reset() { void Hand::simulate(float deltaTime, bool isMine) { + if (_isRaveGloveActive) { updateRaveGloveParticles(deltaTime); } @@ -120,9 +121,9 @@ void Hand::render(bool lookingInMirror) { glEnable(GL_RESCALE_NORMAL); if ( SHOW_LEAP_HAND ) { - renderLeapHands(); - //renderFingerTrails(); - //renderHandSpheres(); + //renderLeapHands(); + renderLeapFingerTrails(); + renderLeapHandSpheres(); } } @@ -155,7 +156,6 @@ void Hand::renderRaveGloveStage() { } - void Hand::renderLeapHands() { for (size_t i = 0; i < getNumPalms(); ++i) { PalmData& hand = getPalms()[i]; @@ -165,7 +165,6 @@ void Hand::renderLeapHands() { } } - void Hand::renderLeapHand(PalmData& hand) { glPushMatrix(); @@ -265,6 +264,7 @@ void Hand::renderLeapFingerTrails() { } } + void Hand::setLeapHands(const std::vector& handPositions, const std::vector& handNormals) { for (size_t i = 0; i < getNumPalms(); ++i) { diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 65ae0c6952..9c254453e1 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -130,6 +130,8 @@ int AvatarData::getBroadcastData(unsigned char* destinationBuffer) { // leap hand data std::vector fingerVectors; + +//printf("about to call _handData->encodeRemoteData(fingerVectors);\n"); _handData->encodeRemoteData(fingerVectors); if (fingerVectors.size() > 255) @@ -244,17 +246,32 @@ int AvatarData::parseData(unsigned char* sourceBuffer, int numBytes) { // hand state, stored as a semi-nibble in the bitItems _handState = getSemiNibbleAt(bitItems,HAND_STATE_START_BIT); +//printf("about to call leap hand data code in AvatarData::parseData...\n"); + // leap hand data if (sourceBuffer - startPosition < numBytes) { + +//printf("got inside of 'if (sourceBuffer - startPosition < numBytes)'\n"); + + // check passed, bytes match unsigned int numFingerVectors = *sourceBuffer++; + +//printf("numFingerVectors = %d\n", numFingerVectors); + + if (numFingerVectors > 0) { + +//printf("ok, we got fingers in AvatarData::parseData\n"); + std::vector fingerVectors(numFingerVectors); for (size_t i = 0; i < numFingerVectors; ++i) { sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].x), fingerVectorRadix); sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].y), fingerVectorRadix); sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].z), fingerVectorRadix); } + +//printf("about to call _handData->decodeRemoteData(fingerVectors);\n"); _handData->decodeRemoteData(fingerVectors); } } diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp index 7bb6eebdc0..605442c926 100755 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -51,12 +51,15 @@ _owningHandData(owningHandData) void HandData::encodeRemoteData(std::vector& fingerVectors) { fingerVectors.clear(); + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& palm = getPalms()[i]; fingerVectors.push_back(palm.getRawPosition()); fingerVectors.push_back(palm.getRawNormal()); for (size_t f = 0; f < palm.getNumFingers(); ++f) { FingerData& finger = palm.getFingers()[f]; + if (finger.isActive()) { fingerVectors.push_back(finger.getTipRawPosition()); fingerVectors.push_back(finger.getRootRawPosition()); @@ -82,7 +85,7 @@ void HandData::decodeRemoteData(const std::vector& fingerVectors) { if (palmActive) { palm.setRawPosition(fingerVectors[vectorIndex++]); palm.setRawNormal(fingerVectors[vectorIndex++]); - for (size_t f = 0; f < NUM_FINGERS_PER_HAND; ++f) { + for (size_t f = 0; f < NUM_FINGERS_PER_HAND; ++f) { FingerData& finger = palm.getFingers()[i]; finger.setRawTipPosition(fingerVectors[vectorIndex++]); finger.setRawRootPosition(fingerVectors[vectorIndex++]); From c60acfad28f2b8db157fc21e8e852786dfc63580 Mon Sep 17 00:00:00 2001 From: Jeffrey Ventrella Date: Fri, 2 Aug 2013 11:00:53 -0700 Subject: [PATCH 24/26] merge --- libraries/avatars/src/HandData.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) mode change 100755 => 100644 libraries/avatars/src/HandData.cpp diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp old mode 100755 new mode 100644 index 541c223e4e..0b91191d92 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -88,8 +88,9 @@ void HandData::decodeRemoteData(const std::vector& fingerVectors) { if (palmActive) { palm.setRawPosition(fingerVectors[vectorIndex++]); palm.setRawNormal(fingerVectors[vectorIndex++]); - for (size_t f = 0; f < NUM_FINGERS_PER_HAND; ++f) { - FingerData& finger = palm.getFingers()[i]; + for (size_t f = 0; f < NUM_FINGERS_PER_HAND; ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.setActive(true); finger.setRawTipPosition(fingerVectors[vectorIndex++]); finger.setRawRootPosition(fingerVectors[vectorIndex++]); } From a9ec94de492380c11209c2049107eb23c62a9c96 Mon Sep 17 00:00:00 2001 From: Jeffrey Ventrella Date: Fri, 2 Aug 2013 13:35:35 -0700 Subject: [PATCH 25/26] fixed a problem with emitters not being coordinated with Leap fingers --- interface/src/avatar/Hand.cpp | 119 ++++++++++++++++------------------ interface/src/avatar/Hand.h | 8 ++- 2 files changed, 61 insertions(+), 66 deletions(-) diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 7e5652c5d5..e5afe7e654 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -64,7 +64,8 @@ void Hand::calculateGeometry() { _basePosition = head.getPosition() + head.getOrientation() * offset; _baseOrientation = head.getOrientation(); - _leapBalls.clear(); + // generate finger tip balls.... + _leapFingerTipBalls.clear(); for (size_t i = 0; i < getNumPalms(); ++i) { PalmData& palm = getPalms()[i]; if (palm.isActive()) { @@ -72,8 +73,8 @@ void Hand::calculateGeometry() { FingerData& finger = palm.getFingers()[f]; if (finger.isActive()) { const float standardBallRadius = 0.01f; - _leapBalls.resize(_leapBalls.size() + 1); - HandBall& ball = _leapBalls.back(); + _leapFingerTipBalls.resize(_leapFingerTipBalls.size() + 1); + HandBall& ball = _leapFingerTipBalls.back(); ball.rotation = _baseOrientation; ball.position = finger.getTipPosition(); ball.radius = standardBallRadius; @@ -83,6 +84,27 @@ void Hand::calculateGeometry() { } } } + + // generate finger rot balls.... + _leapFingerRootBalls.clear(); + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& palm = getPalms()[i]; + if (palm.isActive()) { + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + if (finger.isActive()) { + const float standardBallRadius = 0.01f; + _leapFingerRootBalls.resize(_leapFingerRootBalls.size() + 1); + HandBall& ball = _leapFingerRootBalls.back(); + ball.rotation = _baseOrientation; + ball.position = finger.getRootPosition(); + ball.radius = standardBallRadius; + ball.touchForce = 0.0; + ball.isCollidable = true; + } + } + } + } } void Hand::setRaveGloveEffectsMode(QKeyEvent* event) { @@ -191,15 +213,28 @@ void Hand::renderLeapHand(PalmData& hand) { void Hand::renderLeapHandSpheres() { glPushMatrix(); // Draw the leap balls - for (size_t i = 0; i < _leapBalls.size(); i++) { + for (size_t i = 0; i < _leapFingerTipBalls.size(); i++) { float alpha = 1.0f; if (alpha > 0.0f) { glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, alpha); glPushMatrix(); - glTranslatef(_leapBalls[i].position.x, _leapBalls[i].position.y, _leapBalls[i].position.z); - glutSolidSphere(_leapBalls[i].radius, 20.0f, 20.0f); + glTranslatef(_leapFingerTipBalls[i].position.x, _leapFingerTipBalls[i].position.y, _leapFingerTipBalls[i].position.z); + glutSolidSphere(_leapFingerTipBalls[i].radius, 20.0f, 20.0f); + glPopMatrix(); + } + } + + for (size_t i = 0; i < _leapFingerRootBalls.size(); i++) { + float alpha = 1.0f; + + if (alpha > 0.0f) { + glColor4f(0.3f, 0.4f, 0.6f, alpha); + + glPushMatrix(); + glTranslatef(_leapFingerRootBalls[i].position.x, _leapFingerRootBalls[i].position.y, _leapFingerRootBalls[i].position.z); + glutSolidSphere(_leapFingerRootBalls[i].radius, 20.0f, 20.0f); glPopMatrix(); } } @@ -281,69 +316,27 @@ void Hand::setLeapHands(const std::vector& handPositions, } -// call this right after the geometry of the leap hands are set +// call this soon after the geometry of the leap hands are set void Hand::updateRaveGloveEmitters() { - bool debug = false; + for (size_t i = 0; i < NUM_FINGERS; i++) { + _raveGloveParticleSystem.setEmitterActive(_raveGloveEmitter[i], false); + } - if (_raveGloveInitialized) { - - if(debug) printf( "\n" ); - if(debug) printf( "------------------------------------\n" ); - if(debug) printf( "updating rave glove emitters: \n" ); - if(debug) printf( "------------------------------------\n" ); - - int emitterIndex = 0; - - for (size_t i = 0; i < getNumPalms(); ++i) { - PalmData& palm = getPalms()[i]; + for (size_t i = 0; i < _leapFingerTipBalls.size(); i++) { + if (i < NUM_FINGERS) { + glm::vec3 fingerDirection = _leapFingerTipBalls[i].position - _leapFingerRootBalls[i].position; + float fingerLength = glm::length(fingerDirection); - if(debug) printf( "\n" ); - if(debug) printf( "palm %d ", (int)i ); - - if (palm.isActive()) { - - if(debug) printf( "is active\n" ); - - for (size_t f = 0; f < palm.getNumFingers(); ++f) { - FingerData& finger = palm.getFingers()[f]; - - if(debug) printf( "emitterIndex %d: ", emitterIndex ); - - if ((emitterIndex >=0) - && (emitterIndex < NUM_FINGERS)) { - - _raveGloveParticleSystem.setEmitterActive(_raveGloveEmitter[emitterIndex], false); // set to false by default... - - if (finger.isActive()) { - _raveGloveParticleSystem.setEmitterActive(_raveGloveEmitter[emitterIndex], true); - - if(debug) printf( "_raveGloveEmitter[%d] = %d\n", emitterIndex, _raveGloveEmitter[emitterIndex] ); - - glm::vec3 fingerDirection = finger.getTipPosition() - finger.getRootPosition(); - float fingerLength = glm::length(fingerDirection); - - if (fingerLength > 0.0f) { - fingerDirection /= fingerLength; - } else { - fingerDirection = IDENTITY_UP; - } - - assert(_raveGloveEmitter[emitterIndex] >=0 ); - assert(_raveGloveEmitter[emitterIndex] < NUM_FINGERS ); - - _raveGloveParticleSystem.setEmitterPosition (_raveGloveEmitter[emitterIndex], finger.getTipPosition()); - _raveGloveParticleSystem.setEmitterDirection(_raveGloveEmitter[emitterIndex], fingerDirection); - } - } else { - if(debug) printf( "BOGUS finger\n" ); - } - - emitterIndex ++; - } + if (fingerLength > 0.0f) { + fingerDirection /= fingerLength; } else { - if(debug) printf( "is NOT active\n" ); + fingerDirection = IDENTITY_UP; } + + _raveGloveParticleSystem.setEmitterActive (_raveGloveEmitter[i], true); + _raveGloveParticleSystem.setEmitterPosition (_raveGloveEmitter[i], _leapFingerTipBalls[i].position); + _raveGloveParticleSystem.setEmitterDirection(_raveGloveEmitter[i], fingerDirection); } } } diff --git a/interface/src/avatar/Hand.h b/interface/src/avatar/Hand.h index 2ee066d7b6..beb7b8d516 100755 --- a/interface/src/avatar/Hand.h +++ b/interface/src/avatar/Hand.h @@ -65,8 +65,9 @@ public: void setRaveGloveEffectsMode(QKeyEvent* event); // getters - const glm::vec3& getLeapBallPosition (int ball) const { return _leapBalls[ball].position;} - bool isRaveGloveActive () const { return _isRaveGloveActive; } + const glm::vec3& getLeapFingerTipBallPosition (int ball) const { return _leapFingerTipBalls [ball].position;} + const glm::vec3& getLeapFingerRootBallPosition(int ball) const { return _leapFingerRootBalls[ball].position;} + bool isRaveGloveActive() const { return _isRaveGloveActive; } private: // disallow copies of the Hand, copy of owning Avatar is disallowed too @@ -84,7 +85,8 @@ private: float _renderAlpha; bool _lookingInMirror; glm::vec3 _ballColor; - std::vector _leapBalls; + std::vector _leapFingerTipBalls; + std::vector _leapFingerRootBalls; // private methods void setLeapHands(const std::vector& handPositions, From 29c5511a543401ee1e0bbf2441cb68f947667d70 Mon Sep 17 00:00:00 2001 From: Jeffrey Ventrella Date: Fri, 2 Aug 2013 13:49:50 -0700 Subject: [PATCH 26/26] test --- interface/src/avatar/Hand.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index e5afe7e654..45a2117285 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -14,7 +14,7 @@ #include "Util.h" #include "renderer/ProgramObject.h" -const bool SHOW_LEAP_HAND = true; +const bool SHOW_LEAP_HAND = false; using namespace std;