From 720ae561df891ae05e91f4b870b9d4472451c3eb Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 6 Aug 2013 13:27:54 -0700 Subject: [PATCH] Working on support for a textured face mode that doesn't rely on the video stream. --- .../resources/shaders/face_textured.frag | 82 +++++ .../resources/shaders/face_textured.vert | 38 ++ interface/src/Application.h | 5 +- interface/src/VoxelSystem.cpp | 29 +- interface/src/VoxelSystem.h | 1 - interface/src/avatar/Face.cpp | 326 ++++++++++-------- interface/src/avatar/Face.h | 21 +- interface/src/renderer/TextureCache.cpp | 45 +++ interface/src/renderer/TextureCache.h | 27 ++ 9 files changed, 390 insertions(+), 184 deletions(-) create mode 100644 interface/resources/shaders/face_textured.frag create mode 100644 interface/resources/shaders/face_textured.vert create mode 100644 interface/src/renderer/TextureCache.cpp create mode 100644 interface/src/renderer/TextureCache.h diff --git a/interface/resources/shaders/face_textured.frag b/interface/resources/shaders/face_textured.frag new file mode 100644 index 0000000000..fefefc6817 --- /dev/null +++ b/interface/resources/shaders/face_textured.frag @@ -0,0 +1,82 @@ +#version 120 + +// +// face_textured.frag +// fragment shader +// +// Created by Andrzej Kapolka on 8/6/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +// the texture coordinate vector from left to right +uniform vec2 texCoordRight; + +// the texture coordinate vector from bottom to the top +uniform vec2 texCoordUp; + +// the permutation/normal texture +uniform sampler2D permutationNormalTexture; + +// the depth texture +uniform sampler2D depthTexture; + +// the position in model space +varying vec3 position; + +// returns the gradient at a single corner of our sampling cube +vec3 grad(vec3 location) { + float p1 = texture2D(permutationNormalTexture, vec2(location.x / 256.0, 0.25)).r; + float p2 = texture2D(permutationNormalTexture, vec2(p1 + location.y / 256.0, 0.25)).r; + return texture2D(permutationNormalTexture, vec2(p2 + location.z / 256.0, 0.75)).xyz * 2.0 - vec3(1.0, 1.0, 1.0); +} + +// returns the perlin noise value for the specified location +float perlin(vec3 location) { + vec3 floors = floor(location); + vec3 ceils = ceil(location); + vec3 fff = grad(floors); + vec3 ffc = grad(vec3(floors.x, floors.y, ceils.z)); + vec3 fcf = grad(vec3(floors.x, ceils.y, floors.z)); + vec3 fcc = grad(vec3(floors.x, ceils.y, ceils.z)); + vec3 cff = grad(vec3(ceils.x, floors.y, floors.z)); + vec3 cfc = grad(vec3(ceils.x, floors.y, ceils.z)); + vec3 ccf = grad(vec3(ceils.x, ceils.y, floors.z)); + vec3 ccc = grad(ceils); + vec3 ffracts = fract(location); + vec3 cfracts = ffracts - vec3(1.0, 1.0, 1.0); + vec3 params = ffracts*ffracts*(3.0 - 2.0*ffracts); + + float fffv = dot(fff, ffracts); + float ffcv = dot(ffc, vec3(ffracts.x, ffracts.y, cfracts.z)); + float fcfv = dot(fcf, vec3(ffracts.x, cfracts.y, ffracts.z)); + float fccv = dot(fcc, vec3(ffracts.x, cfracts.y, cfracts.z)); + float cffv = dot(cff, vec3(cfracts.x, ffracts.y, ffracts.z)); + float cfcv = dot(cfc, vec3(cfracts.x, ffracts.y, cfracts.z)); + float ccfv = dot(ccf, vec3(cfracts.x, cfracts.y, ffracts.z)); + float cccv = dot(ccc, cfracts); + + return mix( + mix(mix(fffv, cffv, params.x), mix(fcfv, ccfv, params.x), params.y), + mix(mix(ffcv, cfcv, params.x), mix(fccv, cccv, params.x), params.y), + params.z); +} + +void main(void) { + // compute normal from adjacent depth values + float left = texture2D(depthTexture, gl_TexCoord[0].st - texCoordRight * 0.01).r; + float right = texture2D(depthTexture, gl_TexCoord[0].st + texCoordRight * 0.01).r; + float bottom = texture2D(depthTexture, gl_TexCoord[0].st - texCoordUp * 0.01).r; + float top = texture2D(depthTexture, gl_TexCoord[0].st + texCoordUp * 0.01).r; + vec3 normal = normalize(gl_NormalMatrix * vec3(left - right, top - bottom, -0.05)); + + // compute the specular component (sans exponent) based on the normal OpenGL lighting model + float specular = max(0.0, dot(normalize(gl_LightSource[0].position.xyz + vec3(0.0, 0.0, 1.0)), normal)); + + vec3 color = mix(vec3(1.0, 1.0, 1.0), vec3(0.75, 0.75, 0.75), + sin(dot(position, vec3(25.0, 25.0, 25.0)) + 2.0 * perlin(position * 10.0))); + + // standard lighting + gl_FragColor = vec4(color * ( gl_LightModel.ambient.rgb + /* gl_LightSource[0].ambient.rgb + */ + gl_LightSource[0].diffuse.rgb * max(0.0, dot(normal, gl_LightSource[0].position.xyz))) + + pow(specular, gl_FrontMaterial.shininess) * gl_FrontLightProduct[0].specular.rgb, gl_Color.a); +} diff --git a/interface/resources/shaders/face_textured.vert b/interface/resources/shaders/face_textured.vert new file mode 100644 index 0000000000..28bda9a84d --- /dev/null +++ b/interface/resources/shaders/face_textured.vert @@ -0,0 +1,38 @@ +#version 120 + +// +// face_textured.vert +// vertex shader +// +// Created by Andrzej Kapolka on 8/6/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +// the lower left texture coordinate +uniform vec2 texCoordCorner; + +// the texture coordinate vector from left to right +uniform vec2 texCoordRight; + +// the texture coordinate vector from bottom to the top +uniform vec2 texCoordUp; + +// the depth texture +uniform sampler2D depthTexture; + +// the position in model space +varying vec3 position; + +void main(void) { + gl_TexCoord[0] = vec4(texCoordCorner + gl_Vertex.x * texCoordRight + gl_Vertex.y * texCoordUp, 0.0, 1.0); + float depth = texture2D(depthTexture, gl_TexCoord[0].st).r; + + // store the model space vertex + position = gl_Vertex.xyz; + + // set alpha to zero for invalid depth values + const float MIN_VISIBLE_DEPTH = 1.0 / 255.0; + const float MAX_VISIBLE_DEPTH = 254.0 / 255.0; + gl_FrontColor = vec4(1.0, 1.0, 1.0, step(MIN_VISIBLE_DEPTH, depth) * (1.0 - step(MAX_VISIBLE_DEPTH, depth))); + gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, gl_Vertex.y - 0.5, depth - 0.5, 1.0); +} diff --git a/interface/src/Application.h b/interface/src/Application.h index cb851c1dfc..3621f04a94 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -31,7 +31,6 @@ #include "Environment.h" #include "PacketHeaders.h" #include "ParticleSystem.h" -#include "renderer/GeometryCache.h" #include "SerialInterface.h" #include "Stars.h" #include "Swatch.h" @@ -42,6 +41,8 @@ #include "PieMenu.h" #include "avatar/Avatar.h" #include "avatar/HandControl.h" +#include "renderer/GeometryCache.h" +#include "renderer/TextureCache.h" #include "ui/BandwidthDialog.h" #include "ui/ChatEntry.h" #include "ui/VoxelStatsDialog.h" @@ -105,6 +106,7 @@ public: QNetworkAccessManager* getNetworkAccessManager() { return _networkAccessManager; } GeometryCache* getGeometryCache() { return &_geometryCache; } + TextureCache* getTextureCache() { return &_textureCache; } void resetSongMixMenuItem(); @@ -419,6 +421,7 @@ private: int _hmdWarpParamLocation; GeometryCache _geometryCache; + TextureCache _textureCache; ParticleSystem _particleSystem; diff --git a/interface/src/VoxelSystem.cpp b/interface/src/VoxelSystem.cpp index 65d819eddc..aed01412f7 100644 --- a/interface/src/VoxelSystem.cpp +++ b/interface/src/VoxelSystem.cpp @@ -17,8 +17,6 @@ #include // to load voxels from file #include -#include - #include #include #include @@ -493,7 +491,6 @@ glm::vec3 VoxelSystem::computeVoxelVertex(const glm::vec3& startVertex, float vo } ProgramObject* VoxelSystem::_perlinModulateProgram = 0; -GLuint VoxelSystem::_permutationNormalTextureID = 0; void VoxelSystem::init() { @@ -585,29 +582,9 @@ void VoxelSystem::init() { _perlinModulateProgram->addShaderFromSourceFile(QGLShader::Fragment, "resources/shaders/perlin_modulate.frag"); _perlinModulateProgram->link(); + _perlinModulateProgram->bind(); _perlinModulateProgram->setUniformValue("permutationNormalTexture", 0); - - // create the permutation/normal texture - glGenTextures(1, &_permutationNormalTextureID); - glBindTexture(GL_TEXTURE_2D, _permutationNormalTextureID); - - // the first line consists of random permutation offsets - unsigned char data[256 * 2 * 3]; - for (int i = 0; i < 256 * 3; i++) { - data[i] = rand() % 256; - } - // the next, random unit normals - for (int i = 256 * 3; i < 256 * 3 * 2; i += 3) { - glm::vec3 randvec = glm::sphericalRand(1.0f); - data[i] = ((randvec.x + 1.0f) / 2.0f) * 255.0f; - data[i + 1] = ((randvec.y + 1.0f) / 2.0f) * 255.0f; - data[i + 2] = ((randvec.z + 1.0f) / 2.0f) * 255.0f; - } - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, 256, 2, 0, GL_RGB, GL_UNSIGNED_BYTE, data); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - - glBindTexture(GL_TEXTURE_2D, 0); + _perlinModulateProgram->release(); } void VoxelSystem::updateFullVBOs() { @@ -734,7 +711,7 @@ void VoxelSystem::applyScaleAndBindProgram(bool texture) { if (texture) { _perlinModulateProgram->bind(); - glBindTexture(GL_TEXTURE_2D, _permutationNormalTextureID); + glBindTexture(GL_TEXTURE_2D, Application::getInstance()->getTextureCache()->getPermutationNormalTextureID()); } } diff --git a/interface/src/VoxelSystem.h b/interface/src/VoxelSystem.h index 6c6bf261f9..950993a434 100644 --- a/interface/src/VoxelSystem.h +++ b/interface/src/VoxelSystem.h @@ -193,7 +193,6 @@ private: bool _voxelsDirty; static ProgramObject* _perlinModulateProgram; - static GLuint _permutationNormalTextureID; int _hookID; std::vector _freeIndexes; diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp index ffc88d4b5b..9573cd4e4b 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/Face.cpp @@ -22,10 +22,10 @@ using namespace cv; -ProgramObject* Face::_program = 0; -int Face::_texCoordCornerLocation; -int Face::_texCoordRightLocation; -int Face::_texCoordUpLocation; +ProgramObject* Face::_videoProgram = 0; +Face::Locations Face::_videoProgramLocations; +ProgramObject* Face::_texturedProgram = 0; +Face::Locations Face::_texturedProgramLocations; GLuint Face::_vboID; GLuint Face::_iboID; @@ -71,6 +71,7 @@ void Face::setFrameFromWebcam() { void Face::clearFrame() { _colorTextureID = 0; + _depthTextureID = 0; } int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { @@ -119,120 +120,130 @@ int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { _lastFullFrame = fullFrame; } - if (_colorCodec.name == 0) { - // initialize decoder context - vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0); + // read the color data, if non-empty + Mat color; + const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t)); + size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float)); + if (colorSize > 0) { + if (_colorCodec.name == 0) { + // initialize decoder context + vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0); + } + vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0); + vpx_codec_iter_t iterator = 0; + vpx_image_t* image; + while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { + // convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and + // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor + color.create(image->d_h, image->d_w, CV_8UC3); + uchar* yline = image->planes[0]; + uchar* vline = image->planes[1]; + uchar* uline = image->planes[2]; + const int RED_V_WEIGHT = (int)(1.403 * 256); + const int GREEN_V_WEIGHT = (int)(0.714 * 256); + const int GREEN_U_WEIGHT = (int)(0.344 * 256); + const int BLUE_U_WEIGHT = (int)(1.773 * 256); + for (int i = 0; i < image->d_h; i += 2) { + uchar* ysrc = yline; + uchar* vsrc = vline; + uchar* usrc = uline; + for (int j = 0; j < image->d_w; j += 2) { + uchar* tl = color.ptr(i, j); + uchar* tr = color.ptr(i, j + 1); + uchar* bl = color.ptr(i + 1, j); + uchar* br = color.ptr(i + 1, j + 1); + + int v = *vsrc++ - 128; + int u = *usrc++ - 128; + + int redOffset = (RED_V_WEIGHT * v) >> 8; + int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; + int blueOffset = (BLUE_U_WEIGHT * u) >> 8; + + int ytl = ysrc[0]; + int ytr = ysrc[1]; + int ybl = ysrc[image->w]; + int ybr = ysrc[image->w + 1]; + ysrc += 2; + + tl[0] = saturate_cast(ytl + redOffset); + tl[1] = saturate_cast(ytl - greenOffset); + tl[2] = saturate_cast(ytl + blueOffset); + + tr[0] = saturate_cast(ytr + redOffset); + tr[1] = saturate_cast(ytr - greenOffset); + tr[2] = saturate_cast(ytr + blueOffset); + + bl[0] = saturate_cast(ybl + redOffset); + bl[1] = saturate_cast(ybl - greenOffset); + bl[2] = saturate_cast(ybl + blueOffset); + + br[0] = saturate_cast(ybr + redOffset); + br[1] = saturate_cast(ybr - greenOffset); + br[2] = saturate_cast(ybr + blueOffset); + } + yline += image->stride[0] * 2; + vline += image->stride[1]; + uline += image->stride[2]; + } + } + } else if (_colorCodec.name != 0) { + vpx_codec_destroy(&_colorCodec); + _colorCodec.name = 0; } - size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float)); - const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t)); - vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0); - vpx_codec_iter_t iterator = 0; - vpx_image_t* image; - while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { - // convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and - // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor - Mat color(image->d_h, image->d_w, CV_8UC3); - uchar* yline = image->planes[0]; - uchar* vline = image->planes[1]; - uchar* uline = image->planes[2]; - const int RED_V_WEIGHT = (int)(1.403 * 256); - const int GREEN_V_WEIGHT = (int)(0.714 * 256); - const int GREEN_U_WEIGHT = (int)(0.344 * 256); - const int BLUE_U_WEIGHT = (int)(1.773 * 256); - for (int i = 0; i < image->d_h; i += 2) { - uchar* ysrc = yline; - uchar* vsrc = vline; - uchar* usrc = uline; - for (int j = 0; j < image->d_w; j += 2) { - uchar* tl = color.ptr(i, j); - uchar* tr = color.ptr(i, j + 1); - uchar* bl = color.ptr(i + 1, j); - uchar* br = color.ptr(i + 1, j + 1); - - int v = *vsrc++ - 128; - int u = *usrc++ - 128; - - int redOffset = (RED_V_WEIGHT * v) >> 8; - int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; - int blueOffset = (BLUE_U_WEIGHT * u) >> 8; - - int ytl = ysrc[0]; - int ytr = ysrc[1]; - int ybl = ysrc[image->w]; - int ybr = ysrc[image->w + 1]; - ysrc += 2; - - tl[0] = saturate_cast(ytl + redOffset); - tl[1] = saturate_cast(ytl - greenOffset); - tl[2] = saturate_cast(ytl + blueOffset); - - tr[0] = saturate_cast(ytr + redOffset); - tr[1] = saturate_cast(ytr - greenOffset); - tr[2] = saturate_cast(ytr + blueOffset); - - bl[0] = saturate_cast(ybl + redOffset); - bl[1] = saturate_cast(ybl - greenOffset); - bl[2] = saturate_cast(ybl + blueOffset); - - br[0] = saturate_cast(ybr + redOffset); - br[1] = saturate_cast(ybr - greenOffset); - br[2] = saturate_cast(ybr + blueOffset); - } - yline += image->stride[0] * 2; - vline += image->stride[1]; - uline += image->stride[2]; + // read the depth data, if non-empty + Mat depth; + const uint8_t* depthData = colorData + colorSize; + int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData()); + if (depthSize > 0) { + if (_depthCodec.name == 0) { + // initialize decoder context + vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0); } - Mat depth; - - const uint8_t* depthData = colorData + colorSize; - int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData()); - if (depthSize > 0) { - if (_depthCodec.name == 0) { - // initialize decoder context - vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0); - } - vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0); - vpx_codec_iter_t iterator = 0; - vpx_image_t* image; - while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) { - depth.create(image->d_h, image->d_w, CV_8UC1); - uchar* yline = image->planes[0]; - uchar* vline = image->planes[1]; - const uchar EIGHT_BIT_MAXIMUM = 255; - const uchar MASK_THRESHOLD = 192; - for (int i = 0; i < image->d_h; i += 2) { - uchar* ysrc = yline; - uchar* vsrc = vline; - for (int j = 0; j < image->d_w; j += 2) { - if (*vsrc++ < MASK_THRESHOLD) { - *depth.ptr(i, j) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM; - - } else { - *depth.ptr(i, j) = ysrc[0]; - *depth.ptr(i, j + 1) = ysrc[1]; - *depth.ptr(i + 1, j) = ysrc[image->stride[0]]; - *depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1]; - } - ysrc += 2; + vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0); + vpx_codec_iter_t iterator = 0; + vpx_image_t* image; + while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) { + depth.create(image->d_h, image->d_w, CV_8UC1); + uchar* yline = image->planes[0]; + uchar* vline = image->planes[1]; + const uchar EIGHT_BIT_MAXIMUM = 255; + const uchar MASK_THRESHOLD = 192; + for (int i = 0; i < image->d_h; i += 2) { + uchar* ysrc = yline; + uchar* vsrc = vline; + for (int j = 0; j < image->d_w; j += 2) { + if (*vsrc++ < MASK_THRESHOLD) { + *depth.ptr(i, j) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM; + + } else { + *depth.ptr(i, j) = ysrc[0]; + *depth.ptr(i, j + 1) = ysrc[1]; + *depth.ptr(i + 1, j) = ysrc[image->stride[0]]; + *depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1]; } - yline += image->stride[0] * 2; - vline += image->stride[1]; + ysrc += 2; } + yline += image->stride[0] * 2; + vline += image->stride[1]; } } - QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color), - Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio)); + } else if (_depthCodec.name != 0) { + vpx_codec_destroy(&_depthCodec); + _depthCodec.name = 0; } + QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color), + Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio)); return dataBytes; } bool Face::render(float alpha) { - if (_colorTextureID == 0 || _textureRect.size.area() == 0) { + if (!isActive()) { return false; } glPushMatrix(); @@ -275,20 +286,9 @@ bool Face::render(float alpha) { const int INDICES_PER_TRIANGLE = 3; const int INDEX_COUNT = QUAD_COUNT * TRIANGLES_PER_QUAD * INDICES_PER_TRIANGLE; - if (_program == 0) { - _program = new ProgramObject(); - _program->addShaderFromSourceFile(QGLShader::Vertex, "resources/shaders/face.vert"); - _program->addShaderFromSourceFile(QGLShader::Fragment, "resources/shaders/face.frag"); - _program->link(); - - _program->bind(); - _program->setUniformValue("depthTexture", 0); - _program->setUniformValue("colorTexture", 1); - _program->release(); - - _texCoordCornerLocation = _program->uniformLocation("texCoordCorner"); - _texCoordRightLocation = _program->uniformLocation("texCoordRight"); - _texCoordUpLocation = _program->uniformLocation("texCoordUp"); + if (_videoProgram == 0) { + _videoProgram = loadProgram(QString(), "colorTexture", _videoProgramLocations); + _texturedProgram = loadProgram("_textured", "permutationNormalTexture", _texturedProgramLocations); glGenBuffers(1, &_vboID); glBindBuffer(GL_ARRAY_BUFFER, _vboID); @@ -328,14 +328,23 @@ bool Face::render(float alpha) { glBindTexture(GL_TEXTURE_2D, _depthTextureID); glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, _colorTextureID); - _program->bind(); - _program->setUniformValue(_texCoordCornerLocation, + ProgramObject* program = _videoProgram; + Locations* locations = &_videoProgramLocations; + if (false && _colorTextureID != 0) { + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + + } else { + glBindTexture(GL_TEXTURE_2D, Application::getInstance()->getTextureCache()->getPermutationNormalTextureID()); + program = _texturedProgram; + locations = &_texturedProgramLocations; + } + program->bind(); + program->setUniformValue(locations->texCoordCorner, points[0].x / _textureSize.width, points[0].y / _textureSize.height); - _program->setUniformValue(_texCoordRightLocation, + program->setUniformValue(locations->texCoordRight, (points[3].x - points[0].x) / _textureSize.width, (points[3].y - points[0].y) / _textureSize.height); - _program->setUniformValue(_texCoordUpLocation, + program->setUniformValue(locations->texCoordUp, (points[1].x - points[0].x) / _textureSize.width, (points[1].y - points[0].y) / _textureSize.height); glEnableClientState(GL_VERTEX_ARRAY); glVertexPointer(2, GL_FLOAT, 0, 0); @@ -357,7 +366,7 @@ bool Face::render(float alpha) { glDisableClientState(GL_VERTEX_ARRAY); glBindBuffer(GL_ARRAY_BUFFER, 0); - _program->release(); + program->release(); glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0); @@ -392,32 +401,24 @@ void Face::cycleRenderMode() { } void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) { - if (color.empty()) { - // release our textures, if any; there's no more video - if (_colorTextureID != 0) { - glDeleteTextures(1, &_colorTextureID); - _colorTextureID = 0; + Size2f textureSize; + if (!color.empty()) { + if (_colorTextureID == 0) { + glGenTextures(1, &_colorTextureID); } - if (_depthTextureID != 0) { - glDeleteTextures(1, &_depthTextureID); - _depthTextureID = 0; + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + if (_textureSize.width != color.cols || _textureSize.height != color.rows) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, color.cols, color.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + textureSize = color.size(); + _textureRect = RotatedRect(Point2f(color.cols * 0.5f, color.rows * 0.5f), textureSize, 0.0f); + + } else { + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, color.cols, color.rows, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); } - return; - } - - if (_colorTextureID == 0) { - glGenTextures(1, &_colorTextureID); - } - glBindTexture(GL_TEXTURE_2D, _colorTextureID); - bool recreateTextures = (_textureSize.width != color.cols || _textureSize.height != color.rows); - if (recreateTextures) { - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, color.cols, color.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - _textureSize = color.size(); - _textureRect = RotatedRect(Point2f(color.cols * 0.5f, color.rows * 0.5f), _textureSize, 0.0f); - - } else { - glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, color.cols, color.rows, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); + } else if (_colorTextureID != 0) { + glDeleteTextures(1, &_colorTextureID); + _colorTextureID = 0; } if (!depth.empty()) { @@ -425,20 +426,25 @@ void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRati glGenTextures(1, &_depthTextureID); } glBindTexture(GL_TEXTURE_2D, _depthTextureID); - if (recreateTextures) { + if (_textureSize.width != depth.cols || _textureSize.height != depth.rows) { glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depth.cols, depth.rows, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + textureSize = depth.size(); + _textureRect = RotatedRect(Point2f(depth.cols * 0.5f, depth.rows * 0.5f), textureSize, 0.0f); } else { - glBindTexture(GL_TEXTURE_2D, _depthTextureID); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, depth.cols, depth.rows, GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr()); } + } else if (_depthTextureID != 0) { + glDeleteTextures(1, &_depthTextureID); + _depthTextureID = 0; } glBindTexture(GL_TEXTURE_2D, 0); _aspectRatio = aspectRatio; + _textureSize = textureSize; } void Face::destroyCodecs() { @@ -451,3 +457,21 @@ void Face::destroyCodecs() { _depthCodec.name = 0; } } + +ProgramObject* Face::loadProgram(const QString& suffix, const char* secondTextureUniform, Locations& locations) { + ProgramObject* program = new ProgramObject(); + program->addShaderFromSourceFile(QGLShader::Vertex, "resources/shaders/face" + suffix + ".vert"); + program->addShaderFromSourceFile(QGLShader::Fragment, "resources/shaders/face" + suffix + ".frag"); + program->link(); + + program->bind(); + program->setUniformValue("depthTexture", 0); + program->setUniformValue(secondTextureUniform, 1); + program->release(); + + locations.texCoordCorner = program->uniformLocation("texCoordCorner"); + locations.texCoordRight = program->uniformLocation("texCoordRight"); + locations.texCoordUp = program->uniformLocation("texCoordUp"); + + return program; +} diff --git a/interface/src/avatar/Face.h b/interface/src/avatar/Face.h index 893318f186..5e82148f8d 100644 --- a/interface/src/avatar/Face.h +++ b/interface/src/avatar/Face.h @@ -30,7 +30,8 @@ public: Face(Head* owningHead); ~Face(); - bool isFullFrame() const { return _colorTextureID != 0 && _aspectRatio == FULL_FRAME_ASPECT; } + bool isActive() const { return _colorTextureID != 0 || _depthTextureID != 0; } + bool isFullFrame() const { return isActive() && _aspectRatio == FULL_FRAME_ASPECT; } void setFrameFromWebcam(); void clearFrame(); @@ -69,10 +70,20 @@ private: int _frameCount; int _frameBytesRemaining; - static ProgramObject* _program; - static int _texCoordCornerLocation; - static int _texCoordRightLocation; - static int _texCoordUpLocation; + struct Locations { + int texCoordCorner; + int texCoordRight; + int texCoordUp; + }; + + static ProgramObject* loadProgram(const QString& suffix, const char* secondTextureUniform, Locations& locations); + + static ProgramObject* _videoProgram; + static Locations _videoProgramLocations; + + static ProgramObject* _texturedProgram; + static Locations _texturedProgramLocations; + static GLuint _vboID; static GLuint _iboID; }; diff --git a/interface/src/renderer/TextureCache.cpp b/interface/src/renderer/TextureCache.cpp new file mode 100644 index 0000000000..1c832a648e --- /dev/null +++ b/interface/src/renderer/TextureCache.cpp @@ -0,0 +1,45 @@ +// +// TextureCache.cpp +// interface +// +// Created by Andrzej Kapolka on 8/6/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. + +#include + +#include "TextureCache.h" + +TextureCache::TextureCache() : _permutationNormalTextureID(0) { +} + +TextureCache::~TextureCache() { + if (_permutationNormalTextureID != 0) { + glDeleteTextures(1, &_permutationNormalTextureID); + } +} + +GLuint TextureCache::getPermutationNormalTextureID() { + if (_permutationNormalTextureID == 0) { + glGenTextures(1, &_permutationNormalTextureID); + glBindTexture(GL_TEXTURE_2D, _permutationNormalTextureID); + + // the first line consists of random permutation offsets + unsigned char data[256 * 2 * 3]; + for (int i = 0; i < 256 * 3; i++) { + data[i] = rand() % 256; + } + // the next, random unit normals + for (int i = 256 * 3; i < 256 * 3 * 2; i += 3) { + glm::vec3 randvec = glm::sphericalRand(1.0f); + data[i] = ((randvec.x + 1.0f) / 2.0f) * 255.0f; + data[i + 1] = ((randvec.y + 1.0f) / 2.0f) * 255.0f; + data[i + 2] = ((randvec.z + 1.0f) / 2.0f) * 255.0f; + } + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, 256, 2, 0, GL_RGB, GL_UNSIGNED_BYTE, data); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + + glBindTexture(GL_TEXTURE_2D, 0); + } + return _permutationNormalTextureID; +} diff --git a/interface/src/renderer/TextureCache.h b/interface/src/renderer/TextureCache.h new file mode 100644 index 0000000000..9804f038ba --- /dev/null +++ b/interface/src/renderer/TextureCache.h @@ -0,0 +1,27 @@ +// +// TextureCache.h +// interface +// +// Created by Andrzej Kapolka on 8/6/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__TextureCache__ +#define __interface__TextureCache__ + +#include "InterfaceConfig.h" + +class TextureCache { +public: + + TextureCache(); + ~TextureCache(); + + GLuint getPermutationNormalTextureID(); + +private: + + GLuint _permutationNormalTextureID; +}; + +#endif /* defined(__interface__TextureCache__) */