Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-07 16:50:43 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Switching between textured and untextured modes.

Browse source
This commit is contained in:
Andrzej Kapolka 2013-08-06 16:23:58 -07:00
parent 720ae561df
commit bd3c235fe4
6 changed files with 179 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
interface/resources/shaders/face_textured.frag
View file

@ -72,6 +72,7 @@ void main(void) {
// compute the specular component (sans exponent) based on the normal OpenGL lighting model
float specular = max(0.0, dot(normalize(gl_LightSource[0].position.xyz + vec3(0.0, 0.0, 1.0)), normal));
// the base color is a subtle marble texture produced by modulating the phase of a sine wave by perlin noise
vec3 color = mix(vec3(1.0, 1.0, 1.0), vec3(0.75, 0.75, 0.75),
sin(dot(position, vec3(25.0, 25.0, 25.0)) + 2.0 * perlin(position * 10.0)));

3
interface/src/Application.cpp
View file

@ -1943,8 +1943,9 @@ void Application::initMenu() {
_testPing->setChecked(true);
(_fullScreenMode = optionsMenu->addAction("Fullscreen", this, SLOT(setFullscreen(bool)), Qt::Key_F))->setCheckable(true);
optionsMenu->addAction("Webcam", &_webcam, SLOT(setEnabled(bool)))->setCheckable(true);
optionsMenu->addAction("Toggle Skeleton Tracking", &_webcam, SLOT(setSkeletonTrackingOn(bool)))->setCheckable(true);
optionsMenu->addAction("Skeleton Tracking", &_webcam, SLOT(setSkeletonTrackingOn(bool)))->setCheckable(true);
optionsMenu->addAction("Cycle Webcam Send Mode", _webcam.getGrabber(), SLOT(cycleVideoSendMode()));
optionsMenu->addAction("Webcam Texture", _webcam.getGrabber(), SLOT(setDepthOnly(bool)))->setCheckable(true);
optionsMenu->addAction("Go Home", this, SLOT(goHome()), Qt::CTRL | Qt::Key_G);
QMenu* audioMenu = menuBar->addMenu("Audio");

89
interface/src/Webcam.cpp
View file

@ -71,16 +71,17 @@ void Webcam::reset() {
}
void Webcam::renderPreview(int screenWidth, int screenHeight) {
if (_enabled && _colorTextureID != 0) {
glBindTexture(GL_TEXTURE_2D, _colorTextureID);
if (_enabled) {
glEnable(GL_TEXTURE_2D);
glColor3f(1.0f, 1.0f, 1.0f);
glBegin(GL_QUADS);
const int PREVIEW_HEIGHT = 200;
int previewWidth = _textureSize.width * PREVIEW_HEIGHT / _textureSize.height;
int top = screenHeight - 600;
int left = screenWidth - previewWidth - 10;
if (_colorTextureID != 0) {
glBindTexture(GL_TEXTURE_2D, _colorTextureID);
glBegin(GL_QUADS);
glTexCoord2f(0, 0);
glVertex2f(left, top);
glTexCoord2f(1, 0);
@ -90,6 +91,7 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) {
glTexCoord2f(0, 1);
glVertex2f(left, top + PREVIEW_HEIGHT);
glEnd();
}
if (_depthTextureID != 0) {
glBindTexture(GL_TEXTURE_2D, _depthTextureID);
@ -157,6 +159,7 @@ const float METERS_PER_MM = 1.0f / 1000.0f;
void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midFaceDepth,
float aspectRatio, const RotatedRect& faceRect, bool sending, const JointVector& joints) {
if (!color.empty()) {
IplImage colorImage = color;
glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3);
if (_colorTextureID == 0) {
@ -165,13 +168,16 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, _textureSize.width = colorImage.width, _textureSize.height = colorImage.height,
0, format, GL_UNSIGNED_BYTE, colorImage.imageData);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
qDebug("Capturing video at %gx%g.\n", _textureSize.width, _textureSize.height);
} else {
glBindTexture(GL_TEXTURE_2D, _colorTextureID);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _textureSize.width, _textureSize.height, format,
GL_UNSIGNED_BYTE, colorImage.imageData);
}
} else if (_colorTextureID != 0) {
glDeleteTextures(1, &_colorTextureID);
_colorTextureID = 0;
}
if (!depth.empty()) {
IplImage depthImage = depth;
@ -189,6 +195,9 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _textureSize.width, _textureSize.height, GL_LUMINANCE,
GL_UNSIGNED_BYTE, depthImage.imageData);
}
} else if (_depthTextureID != 0) {
glDeleteTextures(1, &_depthTextureID);
_depthTextureID = 0;
}
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
glBindTexture(GL_TEXTURE_2D, 0);
@ -273,8 +282,8 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midF
QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame()));
}
FrameGrabber::FrameGrabber() : _initialized(false), _videoSendMode(FULL_FRAME_VIDEO), _capture(0), _searchWindow(0, 0, 0, 0),
_smoothedMidFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) {
FrameGrabber::FrameGrabber() : _initialized(false), _videoSendMode(FULL_FRAME_VIDEO), _depthOnly(false), _capture(0),
_searchWindow(0, 0, 0, 0), _smoothedMidFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) {
}
FrameGrabber::~FrameGrabber() {
@ -374,6 +383,11 @@ void FrameGrabber::cycleVideoSendMode() {
destroyCodecs();
}
void FrameGrabber::setDepthOnly(bool depthOnly) {
_depthOnly = depthOnly;
destroyCodecs();
}
void FrameGrabber::reset() {
_searchWindow = cv::Rect(0, 0, 0, 0);
@ -479,7 +493,7 @@ void FrameGrabber::grabFrame() {
encodedWidth = color.cols;
encodedHeight = color.rows;
aspectRatio = FULL_FRAME_ASPECT;
colorBitrateMultiplier = 4.0f;
colorBitrateMultiplier = depthBitrateMultiplier = 4.0f;
} else {
// if we don't have a search window (yet), try using the face cascade
@ -591,10 +605,26 @@ void FrameGrabber::grabFrame() {
depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset);
}
// increment the frame count that identifies frames
_frameCount++;
QByteArray payload;
if (_videoSendMode != NO_VIDEO) {
// start the payload off with the aspect ratio (zero for full frame)
payload.append((const char*)&aspectRatio, sizeof(float));
// prepare the image in which we'll store the data
const int ENCODED_BITS_PER_Y = 8;
const int ENCODED_BITS_PER_VU = 2;
const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU;
const int BITS_PER_BYTE = 8;
_encodedFace.resize(encodedWidth * encodedHeight * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE);
vpx_image_t vpxImage;
vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, encodedWidth, encodedHeight, 1, (unsigned char*)_encodedFace.data());
if (!_depthOnly || depth.empty()) {
if (_colorCodec.name == 0) {
// initialize encoder context(s)
// initialize encoder context
vpx_codec_enc_cfg_t codecConfig;
vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0);
codecConfig.rc_target_bitrate = ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * colorBitrateMultiplier *
@ -602,14 +632,8 @@ void FrameGrabber::grabFrame() {
codecConfig.g_w = encodedWidth;
codecConfig.g_h = encodedHeight;
vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0);
if (!depth.empty()) {
codecConfig.rc_target_bitrate *= depthBitrateMultiplier;
vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0);
}
}
Mat transform;
if (_videoSendMode == FACE_VIDEO) {
// resize/rotate face into encoding rectangle
_faceColor.create(encodedHeight, encodedWidth, CV_8UC3);
@ -621,14 +645,6 @@ void FrameGrabber::grabFrame() {
// convert from RGB to YV12: see http://www.fourcc.org/yuv.php and
// http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor
const int ENCODED_BITS_PER_Y = 8;
const int ENCODED_BITS_PER_VU = 2;
const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU;
const int BITS_PER_BYTE = 8;
_encodedFace.resize(encodedWidth * encodedHeight * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE);
vpx_image_t vpxImage;
vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, encodedWidth, encodedHeight, 1,
(unsigned char*)_encodedFace.data());
uchar* yline = vpxImage.planes[0];
uchar* vline = vpxImage.planes[1];
uchar* uline = vpxImage.planes[2];
@ -676,10 +692,7 @@ void FrameGrabber::grabFrame() {
}
// encode the frame
vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME);
// start the payload off with the aspect ratio (zero for full frame)
payload.append((const char*)&aspectRatio, sizeof(float));
vpx_codec_encode(&_colorCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME);
// extract the encoded frame
vpx_codec_iter_t iterator = 0;
@ -691,8 +704,30 @@ void FrameGrabber::grabFrame() {
payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz);
}
}
} else {
// zero length indicates no color info
const size_t ZERO_SIZE = 0;
payload.append((const char*)&ZERO_SIZE, sizeof(size_t));
// we can use more bits for depth
depthBitrateMultiplier *= 2.0f;
// don't bother reporting the color
color = Mat();
}
if (!depth.empty()) {
if (_depthCodec.name == 0) {
// initialize encoder context
vpx_codec_enc_cfg_t codecConfig;
vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0);
codecConfig.rc_target_bitrate = ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * depthBitrateMultiplier *
codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h;
codecConfig.g_w = encodedWidth;
codecConfig.g_h = encodedHeight;
vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0);
}
// convert with mask
uchar* yline = vpxImage.planes[0];
uchar* vline = vpxImage.planes[1];

2
interface/src/Webcam.h
View file

@ -112,6 +112,7 @@ public:
public slots:
void cycleVideoSendMode();
void setDepthOnly(bool depthOnly);
void reset();
void shutdown();
void grabFrame();
@ -126,6 +127,7 @@ private:
bool _initialized;
VideoSendMode _videoSendMode;
bool _depthOnly;
CvCapture* _capture;
cv::CascadeClassifier _faceCascade;
cv::Mat _hsvFrame;

22
interface/src/avatar/Face.cpp
View file

@ -112,18 +112,20 @@ int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) {
return dataBytes;
}
// the switch from full frame to not (or vice versa) requires us to reinit the codecs
// the switch between full frame or depth only modes requires us to reinit the codecs
float aspectRatio = *(const float*)_arrivingFrame.constData();
size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float));
bool fullFrame = (aspectRatio == FULL_FRAME_ASPECT);
if (fullFrame != _lastFullFrame) {
bool depthOnly = (colorSize == 0);
if (fullFrame != _lastFullFrame || depthOnly != _lastDepthOnly) {
destroyCodecs();
_lastFullFrame = fullFrame;
_lastDepthOnly = depthOnly;
}
// read the color data, if non-empty
Mat color;
const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t));
size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float));
if (colorSize > 0) {
if (_colorCodec.name == 0) {
// initialize decoder context
@ -331,7 +333,7 @@ bool Face::render(float alpha) {
ProgramObject* program = _videoProgram;
Locations* locations = &_videoProgramLocations;
if (false && _colorTextureID != 0) {
if (_colorTextureID != 0) {
glBindTexture(GL_TEXTURE_2D, _colorTextureID);
} else {
@ -401,13 +403,14 @@ void Face::cycleRenderMode() {
}
void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) {
Size2f textureSize;
Size2f textureSize = _textureSize;
if (!color.empty()) {
if (_colorTextureID == 0) {
bool generate = (_colorTextureID == 0);
if (generate) {
glGenTextures(1, &_colorTextureID);
}
glBindTexture(GL_TEXTURE_2D, _colorTextureID);
if (_textureSize.width != color.cols || _textureSize.height != color.rows) {
if (_textureSize.width != color.cols || _textureSize.height != color.rows || generate) {
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, color.cols, color.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, color.ptr());
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
textureSize = color.size();
@ -422,11 +425,12 @@ void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRati
}
if (!depth.empty()) {
if (_depthTextureID == 0) {
bool generate = (_depthTextureID == 0);
if (generate) {
glGenTextures(1, &_depthTextureID);
}
glBindTexture(GL_TEXTURE_2D, _depthTextureID);
if (_textureSize.width != depth.cols || _textureSize.height != depth.rows) {
if (_textureSize.width != depth.cols || _textureSize.height != depth.rows || generate) {
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depth.cols, depth.rows, 0,
GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr());
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);

1
interface/src/avatar/Face.h
View file

@ -65,6 +65,7 @@ private:
vpx_codec_ctx_t _colorCodec;
vpx_codec_ctx_t _depthCodec;
bool _lastFullFrame;
bool _lastDepthOnly;
QByteArray _arrivingFrame;
int _frameCount;