Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-08-09 11:29:50 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'pie_menu' of https://github.com/Atlante45/hifi into stalking_option

Browse source
This commit is contained in:
atlante45 2013-08-02 14:15:07 -07:00
commit f441295eb1
21 changed files with 944 additions and 574 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
interface/resources/shaders/face.vert
View file

@ -17,9 +17,6 @@ uniform vec2 texCoordRight;
// the texture coordinate vector from bottom to the top // the texture coordinate vector from bottom to the top
uniform vec2 texCoordUp; uniform vec2 texCoordUp;
// the aspect ratio of the image
uniform float aspectRatio;
// the depth texture // the depth texture
uniform sampler2D depthTexture; uniform sampler2D depthTexture;
@ -31,6 +28,5 @@ void main(void) {
const float MIN_VISIBLE_DEPTH = 1.0 / 255.0; const float MIN_VISIBLE_DEPTH = 1.0 / 255.0;
const float MAX_VISIBLE_DEPTH = 254.0 / 255.0; const float MAX_VISIBLE_DEPTH = 254.0 / 255.0;
gl_FrontColor = vec4(1.0, 1.0, 1.0, step(MIN_VISIBLE_DEPTH, depth) * (1.0 - step(MAX_VISIBLE_DEPTH, depth))); gl_FrontColor = vec4(1.0, 1.0, 1.0, step(MIN_VISIBLE_DEPTH, depth) * (1.0 - step(MAX_VISIBLE_DEPTH, depth)));
gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, gl_Vertex.y - 0.5, depth - 0.5, 1.0);
(gl_Vertex.y - 0.5) / aspectRatio, depth * 2.0 - 2.0, 1.0);
} }

272
interface/src/Application.cpp
View file

@ -196,8 +196,9 @@ Application::Application(int& argc, char** argv, timeval &startup_time) :
_isTouchPressed(false), _isTouchPressed(false),
_yawFromTouch(0.0f), _yawFromTouch(0.0f),
_pitchFromTouch(0.0f), _pitchFromTouch(0.0f),
_groundPlaneImpact(0.0f),
_mousePressed(false), _mousePressed(false),
_isHoverVoxel(false),
_isHoverVoxelSounding(false),
_mouseVoxelScale(1.0f / 1024.0f), _mouseVoxelScale(1.0f / 1024.0f),
_justEditedVoxel(false), _justEditedVoxel(false),
_isLookingAtOtherAvatar(false), _isLookingAtOtherAvatar(false),
@ -856,6 +857,11 @@ void Application::mouseMoveEvent(QMouseEvent* event) {
} }
} }
const bool MAKE_SOUND_ON_VOXEL_HOVER = false;
const bool MAKE_SOUND_ON_VOXEL_CLICK = true;
const float HOVER_VOXEL_FREQUENCY = 14080.f;
const float HOVER_VOXEL_DECAY = 0.999f;
void Application::mousePressEvent(QMouseEvent* event) { void Application::mousePressEvent(QMouseEvent* event) {
if (activeWindow() == _window) { if (activeWindow() == _window) {
if (event->button() == Qt::LeftButton) { if (event->button() == Qt::LeftButton) {
@ -866,9 +872,19 @@ void Application::mousePressEvent(QMouseEvent* event) {
_mouseVoxelDragging = _mouseVoxel; _mouseVoxelDragging = _mouseVoxel;
_mousePressed = true; _mousePressed = true;
if (!maybeEditVoxelUnderCursor()) { if (!maybeEditVoxelUnderCursor()) {
_pieMenu.mousePressEvent(_mouseX, _mouseY); _pieMenu.mousePressEvent(_mouseX, _mouseY);
} }
if (MAKE_SOUND_ON_VOXEL_CLICK && _isHoverVoxel && !_isHoverVoxelSounding) {
_hoverVoxelOriginalColor[0] = _hoverVoxel.red;
_hoverVoxelOriginalColor[1] = _hoverVoxel.green;
_hoverVoxelOriginalColor[2] = _hoverVoxel.blue;
_hoverVoxelOriginalColor[3] = 1;
_audio.startCollisionSound(1.0, HOVER_VOXEL_FREQUENCY * _hoverVoxel.s * TREE_SCALE, 0.0, HOVER_VOXEL_DECAY);
_isHoverVoxelSounding = true;
}
} else if (event->button() == Qt::RightButton && checkedVoxelModeAction() != 0) { } else if (event->button() == Qt::RightButton && checkedVoxelModeAction() != 0) {
deleteVoxelUnderCursor(); deleteVoxelUnderCursor();
@ -925,8 +941,10 @@ void Application::touchEndEvent(QTouchEvent* event) {
_isTouchPressed = false; _isTouchPressed = false;
} }
const bool USE_MOUSEWHEEL = false;
void Application::wheelEvent(QWheelEvent* event) { void Application::wheelEvent(QWheelEvent* event) {
if (activeWindow() == _window) { // Wheel Events disabled for now because they are also activated by touch look pitch up/down.
if (USE_MOUSEWHEEL && (activeWindow() == _window)) {
if (checkedVoxelModeAction() == 0) { if (checkedVoxelModeAction() == 0) {
event->ignore(); event->ignore();
return; return;
@ -972,14 +990,15 @@ void Application::sendAvatarFaceVideoMessage(int frameCount, const QByteArray& d
int headerSize = packetPosition - packet; int headerSize = packetPosition - packet;
// break the data up into submessages of the maximum size // break the data up into submessages of the maximum size (at least one, for zero-length packets)
*offsetPosition = 0; *offsetPosition = 0;
while (*offsetPosition < data.size()) { do {
int payloadSize = min(data.size() - (int)*offsetPosition, MAX_PACKET_SIZE - headerSize); int payloadSize = min(data.size() - (int)*offsetPosition, MAX_PACKET_SIZE - headerSize);
memcpy(packetPosition, data.constData() + *offsetPosition, payloadSize); memcpy(packetPosition, data.constData() + *offsetPosition, payloadSize);
getInstance()->controlledBroadcastToNodes(packet, headerSize + payloadSize, &NODE_TYPE_AVATAR_MIXER, 1); getInstance()->controlledBroadcastToNodes(packet, headerSize + payloadSize, &NODE_TYPE_AVATAR_MIXER, 1);
*offsetPosition += payloadSize; *offsetPosition += payloadSize;
}
} while (*offsetPosition < data.size());
} }
// Every second, check the frame rates and other stuff // Every second, check the frame rates and other stuff
@ -1502,16 +1521,18 @@ bool Application::sendVoxelsOperation(VoxelNode* node, void* extraData) {
uint64_t now = usecTimestampNow(); uint64_t now = usecTimestampNow();
// dynamically sleep until we need to fire off the next set of voxels // dynamically sleep until we need to fire off the next set of voxels
const uint64_t CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS = 1000 * 5; // 1 packet every 10 milliseconds
uint64_t elapsed = now - args->lastSendTime; uint64_t elapsed = now - args->lastSendTime;
int usecToSleep = CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS - elapsed; int usecToSleep = CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS - elapsed;
if (usecToSleep > 0) { if (usecToSleep > 0) {
qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", //qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n",
args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed, usecToSleep); // args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed, usecToSleep);
Application::getInstance()->timer();
usleep(usecToSleep); usleep(usecToSleep);
} else { } else {
qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", //qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n",
args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed); // args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed);
} }
args->lastSendTime = now; args->lastSendTime = now;
} }
@ -1579,70 +1600,127 @@ void Application::importVoxelsToClipboard() {
void Application::importVoxels() { void Application::importVoxels() {
QString desktopLocation = QStandardPaths::writableLocation(QStandardPaths::DesktopLocation); QString desktopLocation = QStandardPaths::writableLocation(QStandardPaths::DesktopLocation);
QString fileNameString = QFileDialog::getOpenFileName(_glWidget, tr("Import Voxels"), desktopLocation,
QStringList fileNameStringList = QFileDialog::getOpenFileNames(_glWidget, tr("Import Voxels"), desktopLocation,
tr(IMPORT_FILE_TYPES)); tr(IMPORT_FILE_TYPES));
QByteArray fileNameAscii = fileNameString.toLocal8Bit();
const char* fileName = fileNameAscii.data(); // remember the "selected" voxel point before we do any importing...
float originalX = _mouseVoxel.x;
float originalZ = _mouseVoxel.z;
const int PNG_TYPE_NAME_LENGTH = 4;
const int SVO_TYPE_NAME_LENGTH = 4;
const int SCH_TYPE_NAME_LENGTH = 10;
for (int i = 0; i < fileNameStringList.size(); i++) {
QString fileNameString = fileNameStringList.at(i);
QByteArray fileNameAscii = fileNameString.toLocal8Bit();
const char* fileName = fileNameAscii.data();
VoxelTree importVoxels; int fileTypeNameLength = 0;
if (fileNameString.endsWith(".png", Qt::CaseInsensitive)) { VoxelTree importVoxels;
QImage pngImage = QImage(fileName); if (fileNameString.endsWith(".png", Qt::CaseInsensitive)) {
if (pngImage.height() != pngImage.width()) { QImage pngImage = QImage(fileName);
qDebug("ERROR: Bad PNG size: height != width.\n"); fileTypeNameLength = PNG_TYPE_NAME_LENGTH;
return; if (pngImage.height() != pngImage.width()) {
qDebug("ERROR: Bad PNG size: height != width.\n");
return;
}
const uint32_t* pixels;
if (pngImage.format() == QImage::Format_ARGB32) {
pixels = reinterpret_cast<const uint32_t*>(pngImage.constBits());
} else {
QImage tmp = pngImage.convertToFormat(QImage::Format_ARGB32);
pixels = reinterpret_cast<const uint32_t*>(tmp.constBits());
}
importVoxels.readFromSquareARGB32Pixels(pixels, pngImage.height());
} else if (fileNameString.endsWith(".svo", Qt::CaseInsensitive)) {
importVoxels.readFromSVOFile(fileName);
fileTypeNameLength = SVO_TYPE_NAME_LENGTH;
} else if (fileNameString.endsWith(".schematic", Qt::CaseInsensitive)) {
importVoxels.readFromSchematicFile(fileName);
fileTypeNameLength = SCH_TYPE_NAME_LENGTH;
} }
const uint32_t* pixels; int indexOfFirstPeriod = fileNameString.indexOf('.');
if (pngImage.format() == QImage::Format_ARGB32) {
pixels = reinterpret_cast<const uint32_t*>(pngImage.constBits()); QString fileCoord = fileNameString.mid(indexOfFirstPeriod + 1,
fileNameString.length() - indexOfFirstPeriod - fileTypeNameLength - 1);
indexOfFirstPeriod = fileCoord.indexOf('.');
QString columnNumString = fileCoord.right(fileCoord.length() - indexOfFirstPeriod - 1);
QString rowNumString = fileCoord.left(indexOfFirstPeriod);
int columnNum = columnNumString.toFloat();
int rowNum = rowNumString.toFloat();
qDebug("columnNum: %d\t rowNum: %d\n", columnNum, rowNum);
_mouseVoxel.x = originalX + (columnNum - 1) * _mouseVoxel.s;
_mouseVoxel.z = originalZ + (rowNum - 1) * _mouseVoxel.s;
VoxelNode* selectedNode = _voxels.getVoxelAt(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s);
// Recurse the Import Voxels tree, where everything is root relative, and send all the colored voxels to
// the server as an set voxel message, this will also rebase the voxels to the new location
unsigned char* calculatedOctCode = NULL;
SendVoxelsOperationArgs args;
args.lastSendTime = usecTimestampNow();
args.packetsSent = 0;
args.bytesSent = 0;
int numBytesPacketHeader = populateTypeAndVersion(args.messageBuffer, PACKET_TYPE_SET_VOXEL_DESTRUCTIVE);
unsigned short int* sequenceAt = (unsigned short int*)&args.messageBuffer[numBytesPacketHeader];
*sequenceAt = 0;
args.bufferInUse = numBytesPacketHeader + sizeof(unsigned short int); // set to command + sequence
// we only need the selected voxel to get the newBaseOctCode, which we can actually calculate from the
// voxel size/position details.
if (selectedNode) {
args.newBaseOctCode = selectedNode->getOctalCode();
} else { } else {
QImage tmp = pngImage.convertToFormat(QImage::Format_ARGB32); args.newBaseOctCode = calculatedOctCode = pointToVoxel(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s);
pixels = reinterpret_cast<const uint32_t*>(tmp.constBits()); }
qDebug("column:%d, row:%d, voxel:%f,%f,%f,%f\n", columnNum, rowNum, _mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s );
// send the insert/paste of these voxels
importVoxels.recurseTreeWithOperation(sendVoxelsOperation, &args);
// If we have voxels left in the packet, then send the packet
if (args.bufferInUse > (numBytesPacketHeader + sizeof(unsigned short int))) {
controlledBroadcastToNodes(args.messageBuffer, args.bufferInUse, & NODE_TYPE_VOXEL_SERVER, 1);
args.packetsSent++;
args.bytesSent += args.bufferInUse;
uint64_t now = usecTimestampNow();
// dynamically sleep until we need to fire off the next set of voxels
uint64_t elapsed = now - args.lastSendTime;
int usecToSleep = CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS - elapsed;
if (usecToSleep > 0) {
//qDebug("after sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n",
// args.packetsSent, (long long int)args.bytesSent, (long long int)elapsed, usecToSleep);
usleep(usecToSleep);
} else {
//qDebug("after sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n",
// args.packetsSent, (long long int)args.bytesSent, (long long int)elapsed);
}
args.lastSendTime = now;
} }
importVoxels.readFromSquareARGB32Pixels(pixels, pngImage.height()); if (calculatedOctCode) {
} else if (fileNameString.endsWith(".svo", Qt::CaseInsensitive)) { delete[] calculatedOctCode;
importVoxels.readFromSVOFile(fileName); }
} else if (fileNameString.endsWith(".schematic", Qt::CaseInsensitive)) {
importVoxels.readFromSchematicFile(fileName);
} }
VoxelNode* selectedNode = _voxels.getVoxelAt(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s);
// Recurse the Import Voxels tree, where everything is root relative, and send all the colored voxels to
// the server as an set voxel message, this will also rebase the voxels to the new location
unsigned char* calculatedOctCode = NULL;
SendVoxelsOperationArgs args;
args.lastSendTime = usecTimestampNow();
args.packetsSent = 0;
args.bytesSent = 0;
int numBytesPacketHeader = populateTypeAndVersion(args.messageBuffer, PACKET_TYPE_SET_VOXEL_DESTRUCTIVE);
unsigned short int* sequenceAt = (unsigned short int*)&args.messageBuffer[numBytesPacketHeader];
*sequenceAt = 0;
args.bufferInUse = numBytesPacketHeader + sizeof(unsigned short int); // set to command + sequence
// we only need the selected voxel to get the newBaseOctCode, which we can actually calculate from the
// voxel size/position details.
if (selectedNode) {
args.newBaseOctCode = selectedNode->getOctalCode();
} else {
args.newBaseOctCode = calculatedOctCode = pointToVoxel(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s);
}
importVoxels.recurseTreeWithOperation(sendVoxelsOperation, &args);
// If we have voxels left in the packet, then send the packet
if (args.bufferInUse > (numBytesPacketHeader + sizeof(unsigned short int))) {
controlledBroadcastToNodes(args.messageBuffer, args.bufferInUse, & NODE_TYPE_VOXEL_SERVER, 1);
}
if (calculatedOctCode) {
delete[] calculatedOctCode;
}
// restore the main window's active state // restore the main window's active state
_window->activateWindow(); _window->activateWindow();
} }
@ -1735,6 +1813,8 @@ void Application::initMenu() {
_testPing->setChecked(true); _testPing->setChecked(true);
(_fullScreenMode = optionsMenu->addAction("Fullscreen", this, SLOT(setFullscreen(bool)), Qt::Key_F))->setCheckable(true); (_fullScreenMode = optionsMenu->addAction("Fullscreen", this, SLOT(setFullscreen(bool)), Qt::Key_F))->setCheckable(true);
optionsMenu->addAction("Webcam", &_webcam, SLOT(setEnabled(bool)))->setCheckable(true); optionsMenu->addAction("Webcam", &_webcam, SLOT(setEnabled(bool)))->setCheckable(true);
optionsMenu->addAction("Toggle Skeleton Tracking", &_webcam, SLOT(setSkeletonTrackingOn(bool)))->setCheckable(true);
optionsMenu->addAction("Cycle Webcam Send Mode", _webcam.getGrabber(), SLOT(cycleVideoSendMode()));
optionsMenu->addAction("Go Home", this, SLOT(goHome())); optionsMenu->addAction("Go Home", this, SLOT(goHome()));
QMenu* renderMenu = menuBar->addMenu("Render"); QMenu* renderMenu = menuBar->addMenu("Render");
@ -1762,7 +1842,6 @@ void Application::initMenu() {
(_renderLookatIndicatorOn = renderMenu->addAction("Lookat Indicator"))->setCheckable(true); (_renderLookatIndicatorOn = renderMenu->addAction("Lookat Indicator"))->setCheckable(true);
_renderLookatIndicatorOn->setChecked(true); _renderLookatIndicatorOn->setChecked(true);
(_renderParticleSystemOn = renderMenu->addAction("Particle System"))->setCheckable(true); (_renderParticleSystemOn = renderMenu->addAction("Particle System"))->setCheckable(true);
_renderParticleSystemOn->setChecked(true);
(_manualFirstPerson = renderMenu->addAction( (_manualFirstPerson = renderMenu->addAction(
"First Person", this, SLOT(setRenderFirstPerson(bool)), Qt::Key_P))->setCheckable(true); "First Person", this, SLOT(setRenderFirstPerson(bool)), Qt::Key_P))->setCheckable(true);
(_manualThirdPerson = renderMenu->addAction( (_manualThirdPerson = renderMenu->addAction(
@ -2039,6 +2118,7 @@ void Application::renderLookatIndicator(glm::vec3 pointOfInterest, Camera& which
} }
void Application::update(float deltaTime) { void Application::update(float deltaTime) {
// Use Transmitter Hand to move hand if connected, else use mouse // Use Transmitter Hand to move hand if connected, else use mouse
if (_myTransmitter.isConnected()) { if (_myTransmitter.isConnected()) {
const float HAND_FORCE_SCALING = 0.01f; const float HAND_FORCE_SCALING = 0.01f;
@ -2071,11 +2151,46 @@ void Application::update(float deltaTime) {
// If the mouse is over another avatar's head... // If the mouse is over another avatar's head...
glm::vec3 myLookAtFromMouse(eyePosition); glm::vec3 myLookAtFromMouse(eyePosition);
_myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse);
} else if (_isHoverVoxel) {
// Look at the hovered voxel
glm::vec3 lookAtSpot = getMouseVoxelWorldCoordinates(_hoverVoxel);
_myAvatar.getHead().setLookAtPosition(lookAtSpot);
} else { } else {
// Just look in direction of the mouse ray
glm::vec3 myLookAtFromMouse(mouseRayOrigin + mouseRayDirection); glm::vec3 myLookAtFromMouse(mouseRayOrigin + mouseRayDirection);
_myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse);
} }
// Find the voxel we are hovering over, and respond if clicked
float distance;
BoxFace face;
// If we have clicked on a voxel, update it's color
if (_isHoverVoxelSounding) {
VoxelNode* hoveredNode = _voxels.getVoxelAt(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s);
float bright = _audio.getCollisionSoundMagnitude();
nodeColor clickColor = { 255 * bright + _hoverVoxelOriginalColor[0] * (1.f - bright),
_hoverVoxelOriginalColor[1] * (1.f - bright),
_hoverVoxelOriginalColor[2] * (1.f - bright), 1 };
hoveredNode->setColor(clickColor);
if (bright < 0.01f) {
hoveredNode->setColor(_hoverVoxelOriginalColor);
_isHoverVoxelSounding = false;
}
} else {
// Check for a new hover voxel
glm::vec4 oldVoxel(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s);
_isHoverVoxel = _voxels.findRayIntersection(mouseRayOrigin, mouseRayDirection, _hoverVoxel, distance, face);
if (MAKE_SOUND_ON_VOXEL_HOVER && _isHoverVoxel && glm::vec4(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s) != oldVoxel) {
_hoverVoxelOriginalColor[0] = _hoverVoxel.red;
_hoverVoxelOriginalColor[1] = _hoverVoxel.green;
_hoverVoxelOriginalColor[2] = _hoverVoxel.blue;
_hoverVoxelOriginalColor[3] = 1;
_audio.startCollisionSound(1.0, HOVER_VOXEL_FREQUENCY * _hoverVoxel.s * TREE_SCALE, 0.0, HOVER_VOXEL_DECAY);
_isHoverVoxelSounding = true;
}
}
// If we are dragging on a voxel, add thrust according to the amount the mouse is dragging // If we are dragging on a voxel, add thrust according to the amount the mouse is dragging
const float VOXEL_GRAB_THRUST = 0.0f; const float VOXEL_GRAB_THRUST = 0.0f;
if (_mousePressed && (_mouseVoxel.s != 0)) { if (_mousePressed && (_mouseVoxel.s != 0)) {
@ -2101,8 +2216,6 @@ void Application::update(float deltaTime) {
(fabs(_myAvatar.getVelocity().x) + (fabs(_myAvatar.getVelocity().x) +
fabs(_myAvatar.getVelocity().y) + fabs(_myAvatar.getVelocity().y) +
fabs(_myAvatar.getVelocity().z)) / 3 < MAX_AVATAR_EDIT_VELOCITY) { fabs(_myAvatar.getVelocity().z)) / 3 < MAX_AVATAR_EDIT_VELOCITY) {
float distance;
BoxFace face;
if (_voxels.findRayIntersection(mouseRayOrigin, mouseRayDirection, _mouseVoxel, distance, face)) { if (_voxels.findRayIntersection(mouseRayOrigin, mouseRayDirection, _mouseVoxel, distance, face)) {
if (distance < MAX_VOXEL_EDIT_DISTANCE) { if (distance < MAX_VOXEL_EDIT_DISTANCE) {
// find the nearest voxel with the desired scale // find the nearest voxel with the desired scale
@ -2289,20 +2402,18 @@ void Application::update(float deltaTime) {
} }
void Application::updateAvatar(float deltaTime) { void Application::updateAvatar(float deltaTime) {
// When head is rotated via touch/mouse look, slowly turn body to follow // rotate body yaw for yaw received from multitouch
const float BODY_FOLLOW_HEAD_RATE = 0.5f;
// update body yaw by body yaw delta
_myAvatar.setOrientation(_myAvatar.getOrientation() _myAvatar.setOrientation(_myAvatar.getOrientation()
* glm::quat(glm::vec3(0, _yawFromTouch * deltaTime * BODY_FOLLOW_HEAD_RATE, 0) * deltaTime)); * glm::quat(glm::vec3(0, _yawFromTouch * deltaTime, 0)));
_yawFromTouch -= _yawFromTouch * deltaTime * BODY_FOLLOW_HEAD_RATE; _yawFromTouch = 0.f;
// Update my avatar's state from gyros and/or webcam // Update my avatar's state from gyros and/or webcam
_myAvatar.updateFromGyrosAndOrWebcam(_gyroLook->isChecked(), _myAvatar.updateFromGyrosAndOrWebcam(_gyroLook->isChecked(),
glm::vec3(_headCameraPitchYawScale, glm::vec3(_headCameraPitchYawScale,
_headCameraPitchYawScale, _headCameraPitchYawScale,
_headCameraPitchYawScale), _headCameraPitchYawScale),
_yawFromTouch, 0.f,
_pitchFromTouch); _pitchFromTouch);
if (_serialHeadSensor.isActive()) { if (_serialHeadSensor.isActive()) {
@ -2761,8 +2872,12 @@ void Application::displayOverlay() {
glDisable(GL_DEPTH_TEST); glDisable(GL_DEPTH_TEST);
glDisable(GL_LIGHTING); glDisable(GL_LIGHTING);
// Display a single screen-size quad to // Display a single screen-size quad to create an alpha blended 'collision' flash
renderCollisionOverlay(_glWidget->width(), _glWidget->height(), _audio.getCollisionSoundMagnitude()); float collisionSoundMagnitude = _audio.getCollisionSoundMagnitude();
const float VISIBLE_COLLISION_SOUND_MAGNITUDE = 0.5f;
if (collisionSoundMagnitude > VISIBLE_COLLISION_SOUND_MAGNITUDE) {
renderCollisionOverlay(_glWidget->width(), _glWidget->height(), _audio.getCollisionSoundMagnitude());
}
#ifndef _WIN32 #ifndef _WIN32
_audio.render(_glWidget->width(), _glWidget->height()); _audio.render(_glWidget->width(), _glWidget->height());
@ -3501,6 +3616,7 @@ void* Application::networkReceive(void* args) {
} }
if (NodeList::getInstance()->getNodeSocket()->receive(&senderAddress, app->_incomingPacket, &bytesReceived)) { if (NodeList::getInstance()->getNodeSocket()->receive(&senderAddress, app->_incomingPacket, &bytesReceived)) {
app->_packetCount++; app->_packetCount++;
app->_bytesCount += bytesReceived; app->_bytesCount += bytesReceived;
@ -3545,13 +3661,11 @@ void* Application::networkReceive(void* args) {
Node* voxelServer = NodeList::getInstance()->soloNodeOfType(NODE_TYPE_VOXEL_SERVER); Node* voxelServer = NodeList::getInstance()->soloNodeOfType(NODE_TYPE_VOXEL_SERVER);
if (voxelServer && socketMatch(voxelServer->getActiveSocket(), &senderAddress)) { if (voxelServer && socketMatch(voxelServer->getActiveSocket(), &senderAddress)) {
voxelServer->lock(); voxelServer->lock();
if (messageData[0] == PACKET_TYPE_ENVIRONMENT_DATA) { if (messageData[0] == PACKET_TYPE_ENVIRONMENT_DATA) {
app->_environment.parseData(&senderAddress, messageData, messageLength); app->_environment.parseData(&senderAddress, messageData, messageLength);
} else { } else {
app->_voxels.parseData(messageData, messageLength); app->_voxels.parseData(messageData, messageLength);
} }
voxelServer->unlock(); voxelServer->unlock();
} }
} }

10
interface/src/Application.h
View file

@ -110,8 +110,6 @@ public slots:
void sendAvatarFaceVideoMessage(int frameCount, const QByteArray& data); void sendAvatarFaceVideoMessage(int frameCount, const QByteArray& data);
void setGroundPlaneImpact(float groundPlaneImpact) { _groundPlaneImpact = groundPlaneImpact; }
private slots: private slots:
@ -376,14 +374,16 @@ private:
float _yawFromTouch; float _yawFromTouch;
float _pitchFromTouch; float _pitchFromTouch;
float _groundPlaneImpact;
VoxelDetail _mouseVoxelDragging; VoxelDetail _mouseVoxelDragging;
glm::vec3 _voxelThrust; glm::vec3 _voxelThrust;
bool _mousePressed; // true if mouse has been pressed (clear when finished) bool _mousePressed; // true if mouse has been pressed (clear when finished)
VoxelDetail _hoverVoxel; // Stuff about the voxel I am hovering or clicking
bool _isHoverVoxel;
bool _isHoverVoxelSounding;
nodeColor _hoverVoxelOriginalColor;
VoxelDetail _mouseVoxel; // details of the voxel under the mouse cursor VoxelDetail _mouseVoxel; // details of the voxel to be edited
float _mouseVoxelScale; // the scale for adding/removing voxels float _mouseVoxelScale; // the scale for adding/removing voxels
glm::vec3 _lastMouseVoxelPos; // the position of the last mouse voxel edit glm::vec3 _lastMouseVoxelPos; // the position of the last mouse voxel edit
bool _justEditedVoxel; // set when we've just added/deleted/colored a voxel bool _justEditedVoxel; // set when we've just added/deleted/colored a voxel

53
interface/src/ParticleSystem.cpp
View file

@ -27,6 +27,7 @@ ParticleSystem::ParticleSystem() {
for (unsigned int emitterIndex = 0; emitterIndex < MAX_EMITTERS; emitterIndex++) { for (unsigned int emitterIndex = 0; emitterIndex < MAX_EMITTERS; emitterIndex++) {
Emitter * e = &_emitter[emitterIndex]; Emitter * e = &_emitter[emitterIndex];
e->active = false;
e->position = glm::vec3(0.0f, 0.0f, 0.0f); e->position = glm::vec3(0.0f, 0.0f, 0.0f);
e->previousPosition = glm::vec3(0.0f, 0.0f, 0.0f); e->previousPosition = glm::vec3(0.0f, 0.0f, 0.0f);
e->direction = glm::vec3(0.0f, 1.0f, 0.0f); e->direction = glm::vec3(0.0f, 1.0f, 0.0f);
@ -72,25 +73,16 @@ void ParticleSystem::simulate(float deltaTime) {
_timer += deltaTime; _timer += deltaTime;
// emit particles // update emitters
for (int e = 0; e < _numEmitters; e++) { for (int emitterIndex = 0; emitterIndex < _numEmitters; emitterIndex++) {
assert(emitterIndex <= MAX_EMITTERS);
assert(e >= 0); if (_emitter[emitterIndex].active) {
assert(e <= MAX_EMITTERS); updateEmitter(emitterIndex, deltaTime);
assert(_emitter[e].rate >= 0);
_emitter[e].emitReserve += _emitter[e].rate * deltaTime;
_emitter[e].numParticlesEmittedThisTime = (int)_emitter[e].emitReserve;
_emitter[e].emitReserve -= _emitter[e].numParticlesEmittedThisTime;
for (int p = 0; p < _emitter[e].numParticlesEmittedThisTime; p++) {
float timeFraction = (float)p / (float)_emitter[e].numParticlesEmittedThisTime;
createParticle(e, timeFraction);
} }
} }
// update particles
// update particles
for (int p = 0; p < MAX_PARTICLES; p++) { for (int p = 0; p < MAX_PARTICLES; p++) {
if (_particle[p].alive) { if (_particle[p].alive) {
if (_particle[p].age > _emitter[_particle[p].emitterIndex].particleLifespan) { if (_particle[p].age > _emitter[_particle[p].emitterIndex].particleLifespan) {
@ -102,6 +94,20 @@ void ParticleSystem::simulate(float deltaTime) {
} }
} }
void ParticleSystem::updateEmitter(int emitterIndex, float deltaTime) {
_emitter[emitterIndex].emitReserve += _emitter[emitterIndex].rate * deltaTime;
_emitter[emitterIndex].numParticlesEmittedThisTime = (int)_emitter[emitterIndex].emitReserve;
_emitter[emitterIndex].emitReserve -= _emitter[emitterIndex].numParticlesEmittedThisTime;
for (int p = 0; p < _emitter[emitterIndex].numParticlesEmittedThisTime; p++) {
float timeFraction = (float)p / (float)_emitter[emitterIndex].numParticlesEmittedThisTime;
createParticle(emitterIndex, timeFraction);
}
}
void ParticleSystem::createParticle(int e, float timeFraction) { void ParticleSystem::createParticle(int e, float timeFraction) {
for (unsigned int p = 0; p < MAX_PARTICLES; p++) { for (unsigned int p = 0; p < MAX_PARTICLES; p++) {
@ -212,7 +218,6 @@ void ParticleSystem::setParticleAttributes(int emitterIndex, ParticleLifeStage l
} }
void ParticleSystem::updateParticle(int p, float deltaTime) { void ParticleSystem::updateParticle(int p, float deltaTime) {
Emitter myEmitter = _emitter[_particle[p].emitterIndex]; Emitter myEmitter = _emitter[_particle[p].emitterIndex];
@ -363,14 +368,16 @@ void ParticleSystem::killAllParticles() {
void ParticleSystem::render() { void ParticleSystem::render() {
// render the emitters // render the emitters
for (int e = 0; e < _numEmitters; e++) { for (int e = 0; e < MAX_EMITTERS; e++) {
if (_emitter[e].showingBaseParticle) { if (_emitter[e].active) {
glColor4f(_particle[0].color.r, _particle[0].color.g, _particle[0].color.b, _particle[0].color.a); if (_emitter[e].showingBaseParticle) {
glPushMatrix(); glColor4f(_particle[0].color.r, _particle[0].color.g, _particle[0].color.b, _particle[0].color.a);
glTranslatef(_emitter[e].position.x, _emitter[e].position.y, _emitter[e].position.z); glPushMatrix();
glutSolidSphere(_particle[0].radius, _emitter[e].particleResolution, _emitter[e].particleResolution); glTranslatef(_emitter[e].position.x, _emitter[e].position.y, _emitter[e].position.z);
glPopMatrix(); glutSolidSphere(_particle[0].radius, _emitter[e].particleResolution, _emitter[e].particleResolution);
glPopMatrix();
}
} }
if (_emitter[e].visible) { if (_emitter[e].visible) {

5
interface/src/ParticleSystem.h
View file

@ -10,10 +10,10 @@
#include <glm/gtc/quaternion.hpp> #include <glm/gtc/quaternion.hpp>
const int MAX_PARTICLES = 5000;
const int NULL_EMITTER = -1; const int NULL_EMITTER = -1;
const int NULL_PARTICLE = -1; const int NULL_PARTICLE = -1;
const int MAX_EMITTERS = 100; const int MAX_EMITTERS = 100;
const int MAX_PARTICLES = 5000;
enum ParticleRenderStyle enum ParticleRenderStyle
{ {
@ -78,6 +78,7 @@ public:
void setParticleAttributes (int emitterIndex, ParticleAttributes attributes); // set attributes for whole life of particles void setParticleAttributes (int emitterIndex, ParticleAttributes attributes); // set attributes for whole life of particles
void setParticleAttributes (int emitterIndex, ParticleLifeStage lifeStage, ParticleAttributes attributes); // set attributes for this life stage void setParticleAttributes (int emitterIndex, ParticleLifeStage lifeStage, ParticleAttributes attributes); // set attributes for this life stage
void setEmitterPosition (int emitterIndex, glm::vec3 position ); void setEmitterPosition (int emitterIndex, glm::vec3 position );
void setEmitterActive (int emitterIndex, bool active ) {_emitter[emitterIndex].active = active; }
void setEmitterParticleResolution (int emitterIndex, int resolution ) {_emitter[emitterIndex].particleResolution = resolution; } void setEmitterParticleResolution (int emitterIndex, int resolution ) {_emitter[emitterIndex].particleResolution = resolution; }
void setEmitterDirection (int emitterIndex, glm::vec3 direction ) {_emitter[emitterIndex].direction = direction; } void setEmitterDirection (int emitterIndex, glm::vec3 direction ) {_emitter[emitterIndex].direction = direction; }
void setShowingEmitter (int emitterIndex, bool showing ) {_emitter[emitterIndex].visible = showing; } void setShowingEmitter (int emitterIndex, bool showing ) {_emitter[emitterIndex].visible = showing; }
@ -101,6 +102,7 @@ private:
}; };
struct Emitter { struct Emitter {
bool active; // if false, the emitter is disabled - allows for easy switching on and off
glm::vec3 position; // the position of the emitter in world coordinates glm::vec3 position; // the position of the emitter in world coordinates
glm::vec3 previousPosition; // the position of the emitter in the previous time step glm::vec3 previousPosition; // the position of the emitter in the previous time step
glm::vec3 direction; // a normalized vector used as an axis for particle emission and other effects glm::vec3 direction; // a normalized vector used as an axis for particle emission and other effects
@ -124,6 +126,7 @@ private:
float _timer; float _timer;
// private methods // private methods
void updateEmitter(int emitterIndex, float deltaTime);
void updateParticle(int index, float deltaTime); void updateParticle(int index, float deltaTime);
void createParticle(int e, float timeFraction); void createParticle(int e, float timeFraction);
void killParticle(int p); void killParticle(int p);

4
interface/src/Physics.cpp
View file

@ -38,3 +38,7 @@ void applyDamping(float deltaTime, glm::vec3& velocity, float linearStrength, fl
} }
} }
void applyDampedSpring(float deltaTime, glm::vec3& velocity, glm::vec3& position, glm::vec3& targetPosition, float k, float damping) {
}

8
interface/src/Util.cpp
View file

@ -339,6 +339,7 @@ void renderCollisionOverlay(int width, int height, float magnitude) {
} }
void renderGroundPlaneGrid(float size, float impact) { void renderGroundPlaneGrid(float size, float impact) {
float IMPACT_SOUND_MAGNITUDE_FOR_RECOLOR = 1.f;
glLineWidth(2.0); glLineWidth(2.0);
glm::vec4 impactColor(1, 0, 0, 1); glm::vec4 impactColor(1, 0, 0, 1);
glm::vec3 lineColor(0.4, 0.5, 0.3); glm::vec3 lineColor(0.4, 0.5, 0.3);
@ -355,7 +356,12 @@ void renderGroundPlaneGrid(float size, float impact) {
} }
// Draw the floor, colored for recent impact // Draw the floor, colored for recent impact
glm::vec4 floorColor = impact * impactColor + (1.f - impact) * surfaceColor; glm::vec4 floorColor;
if (impact > IMPACT_SOUND_MAGNITUDE_FOR_RECOLOR) {
floorColor = impact * impactColor + (1.f - impact) * surfaceColor;
} else {
floorColor = surfaceColor;
}
glColor4fv(&floorColor.x); glColor4fv(&floorColor.x);
glBegin(GL_QUADS); glBegin(GL_QUADS);
glVertex3f(0, 0, 0); glVertex3f(0, 0, 0);

499
interface/src/Webcam.cpp
View file

@ -19,6 +19,7 @@
#include "Application.h" #include "Application.h"
#include "Webcam.h" #include "Webcam.h"
#include "avatar/Face.h"
using namespace cv; using namespace cv;
using namespace std; using namespace std;
@ -32,7 +33,7 @@ int jointVectorMetaType = qRegisterMetaType<JointVector>("JointVector");
int matMetaType = qRegisterMetaType<Mat>("cv::Mat"); int matMetaType = qRegisterMetaType<Mat>("cv::Mat");
int rotatedRectMetaType = qRegisterMetaType<RotatedRect>("cv::RotatedRect"); int rotatedRectMetaType = qRegisterMetaType<RotatedRect>("cv::RotatedRect");
Webcam::Webcam() : _enabled(false), _active(false), _colorTextureID(0), _depthTextureID(0) { Webcam::Webcam() : _enabled(false), _active(false), _colorTextureID(0), _depthTextureID(0), _skeletonTrackingOn(false) {
// the grabber simply runs as fast as possible // the grabber simply runs as fast as possible
_grabber = new FrameGrabber(); _grabber = new FrameGrabber();
_grabber->moveToThread(&_grabberThread); _grabber->moveToThread(&_grabberThread);
@ -154,8 +155,8 @@ Webcam::~Webcam() {
const float METERS_PER_MM = 1.0f / 1000.0f; const float METERS_PER_MM = 1.0f / 1000.0f;
void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float meanFaceDepth, void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midFaceDepth,
const RotatedRect& faceRect, const JointVector& joints) { float aspectRatio, const RotatedRect& faceRect, bool sending, const JointVector& joints) {
IplImage colorImage = color; IplImage colorImage = color;
glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3);
if (_colorTextureID == 0) { if (_colorTextureID == 0) {
@ -192,9 +193,11 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
glBindTexture(GL_TEXTURE_2D, 0); glBindTexture(GL_TEXTURE_2D, 0);
// store our face rect and joints, update our frame count for fps computation // store our various data, update our frame count for fps computation
_aspectRatio = aspectRatio;
_faceRect = faceRect; _faceRect = faceRect;
_joints = joints; _sending = sending;
_joints = _skeletonTrackingOn ? joints : JointVector();
_frameCount++; _frameCount++;
const int MAX_FPS = 60; const int MAX_FPS = 60;
@ -242,18 +245,18 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean
if (_initialFaceRect.size.area() == 0) { if (_initialFaceRect.size.area() == 0) {
_initialFaceRect = _faceRect; _initialFaceRect = _faceRect;
_estimatedPosition = glm::vec3(); _estimatedPosition = glm::vec3();
_initialFaceDepth = meanFaceDepth; _initialFaceDepth = midFaceDepth;
} else { } else {
float proportion, z; float proportion, z;
if (meanFaceDepth == UNINITIALIZED_FACE_DEPTH) { if (midFaceDepth == UNINITIALIZED_FACE_DEPTH) {
proportion = sqrtf(_initialFaceRect.size.area() / (float)_faceRect.size.area()); proportion = sqrtf(_initialFaceRect.size.area() / (float)_faceRect.size.area());
const float INITIAL_DISTANCE_TO_CAMERA = 0.333f; const float INITIAL_DISTANCE_TO_CAMERA = 0.333f;
z = INITIAL_DISTANCE_TO_CAMERA * proportion - INITIAL_DISTANCE_TO_CAMERA; z = INITIAL_DISTANCE_TO_CAMERA * proportion - INITIAL_DISTANCE_TO_CAMERA;
} else { } else {
z = (meanFaceDepth - _initialFaceDepth) * METERS_PER_MM; z = (midFaceDepth - _initialFaceDepth) * METERS_PER_MM;
proportion = meanFaceDepth / _initialFaceDepth; proportion = midFaceDepth / _initialFaceDepth;
} }
const float POSITION_SCALE = 0.5f; const float POSITION_SCALE = 0.5f;
_estimatedPosition = glm::vec3( _estimatedPosition = glm::vec3(
@ -270,8 +273,8 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean
QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame())); QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame()));
} }
FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), FrameGrabber::FrameGrabber() : _initialized(false), _videoSendMode(FULL_FRAME_VIDEO), _capture(0), _searchWindow(0, 0, 0, 0),
_smoothedMeanFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) { _smoothedMidFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) {
} }
FrameGrabber::~FrameGrabber() { FrameGrabber::~FrameGrabber() {
@ -364,6 +367,13 @@ static void XN_CALLBACK_TYPE calibrationCompleted(SkeletonCapability& capability
} }
#endif #endif
void FrameGrabber::cycleVideoSendMode() {
_videoSendMode = (VideoSendMode)((_videoSendMode + 1) % VIDEO_SEND_MODE_COUNT);
_searchWindow = cv::Rect(0, 0, 0, 0);
destroyCodecs();
}
void FrameGrabber::reset() { void FrameGrabber::reset() {
_searchWindow = cv::Rect(0, 0, 0, 0); _searchWindow = cv::Rect(0, 0, 0, 0);
@ -379,14 +389,7 @@ void FrameGrabber::shutdown() {
cvReleaseCapture(&_capture); cvReleaseCapture(&_capture);
_capture = 0; _capture = 0;
} }
if (_colorCodec.name != 0) { destroyCodecs();
vpx_codec_destroy(&_colorCodec);
_colorCodec.name = 0;
}
if (_depthCodec.name != 0) {
vpx_codec_destroy(&_depthCodec);
_depthCodec.name = 0;
}
_initialized = false; _initialized = false;
thread()->quit(); thread()->quit();
@ -462,218 +465,208 @@ void FrameGrabber::grabFrame() {
color = image; color = image;
} }
// if we don't have a search window (yet), try using the face cascade int encodedWidth;
int channels = 0; int encodedHeight;
float ranges[] = { 0, 180 }; int depthBitrateMultiplier = 1;
const float* range = ranges; Mat faceTransform;
if (_searchWindow.area() == 0) { float aspectRatio;
vector<Rect> faces; if (_videoSendMode == FULL_FRAME_VIDEO) {
_faceCascade.detectMultiScale(color, faces, 1.1, 6); // no need to find the face if we're sending full frame video
if (!faces.empty()) { _smoothedFaceRect = RotatedRect(Point2f(color.cols / 2.0f, color.rows / 2.0f), Size2f(color.cols, color.rows), 0.0f);
_searchWindow = faces.front(); encodedWidth = color.cols;
updateHSVFrame(color, format); encodedHeight = color.rows;
aspectRatio = FULL_FRAME_ASPECT;
Mat faceHsv(_hsvFrame, _searchWindow);
Mat faceMask(_mask, _searchWindow);
int sizes = 30;
calcHist(&faceHsv, 1, &channels, faceMask, _histogram, 1, &sizes, &range);
double min, max;
minMaxLoc(_histogram, &min, &max);
_histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max);
}
}
RotatedRect faceRect;
if (_searchWindow.area() > 0) {
updateHSVFrame(color, format);
calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range);
bitwise_and(_backProject, _mask, _backProject);
faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1));
Rect faceBounds = faceRect.boundingRect();
Rect imageBounds(0, 0, color.cols, color.rows);
_searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds));
}
const int ENCODED_FACE_WIDTH = 128;
const int ENCODED_FACE_HEIGHT = 128;
if (_colorCodec.name == 0) {
// initialize encoder context(s)
vpx_codec_enc_cfg_t codecConfig;
vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0);
codecConfig.rc_target_bitrate = ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT *
codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h;
codecConfig.g_w = ENCODED_FACE_WIDTH;
codecConfig.g_h = ENCODED_FACE_HEIGHT;
vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0);
if (!depth.empty()) {
int DEPTH_BITRATE_MULTIPLIER = 2;
codecConfig.rc_target_bitrate *= 2;
vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0);
}
}
// correct for 180 degree rotations
if (faceRect.angle < -90.0f) {
faceRect.angle += 180.0f;
} else if (faceRect.angle > 90.0f) {
faceRect.angle -= 180.0f;
}
// compute the smoothed face rect
if (_smoothedFaceRect.size.area() == 0) {
_smoothedFaceRect = faceRect;
} else { } else {
const float FACE_RECT_SMOOTHING = 0.9f; // if we don't have a search window (yet), try using the face cascade
_smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING); int channels = 0;
_smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING); float ranges[] = { 0, 180 };
_smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING); const float* range = ranges;
_smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING); if (_searchWindow.area() == 0) {
_smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); vector<Rect> faces;
} _faceCascade.detectMultiScale(color, faces, 1.1, 6);
if (!faces.empty()) {
// resize/rotate face into encoding rectangle _searchWindow = faces.front();
_faceColor.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_8UC3); updateHSVFrame(color, format);
Point2f sourcePoints[4];
_smoothedFaceRect.points(sourcePoints);
Point2f destPoints[] = { Point2f(0, ENCODED_FACE_HEIGHT), Point2f(0, 0), Point2f(ENCODED_FACE_WIDTH, 0) };
Mat transform = getAffineTransform(sourcePoints, destPoints);
warpAffine(color, _faceColor, transform, _faceColor.size());
// convert from RGB to YV12
const int ENCODED_BITS_PER_Y = 8;
const int ENCODED_BITS_PER_VU = 2;
const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU;
const int BITS_PER_BYTE = 8;
_encodedFace.resize(ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE);
vpx_image_t vpxImage;
vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, 1, (unsigned char*)_encodedFace.data());
uchar* yline = vpxImage.planes[0];
uchar* vline = vpxImage.planes[1];
uchar* uline = vpxImage.planes[2];
const int Y_RED_WEIGHT = (int)(0.299 * 256);
const int Y_GREEN_WEIGHT = (int)(0.587 * 256);
const int Y_BLUE_WEIGHT = (int)(0.114 * 256);
const int V_RED_WEIGHT = (int)(0.713 * 256);
const int U_BLUE_WEIGHT = (int)(0.564 * 256);
int redIndex = 0;
int greenIndex = 1;
int blueIndex = 2;
if (format == GL_BGR) {
redIndex = 2;
blueIndex = 0;
}
for (int i = 0; i < ENCODED_FACE_HEIGHT; i += 2) {
uchar* ydest = yline;
uchar* vdest = vline;
uchar* udest = uline;
for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) {
uchar* tl = _faceColor.ptr(i, j);
uchar* tr = _faceColor.ptr(i, j + 1);
uchar* bl = _faceColor.ptr(i + 1, j);
uchar* br = _faceColor.ptr(i + 1, j + 1);
ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8; Mat faceHsv(_hsvFrame, _searchWindow);
ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8; Mat faceMask(_mask, _searchWindow);
ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] * int sizes = 30;
Y_GREEN_WEIGHT + bl[blueIndex] * Y_BLUE_WEIGHT) >> 8; calcHist(&faceHsv, 1, &channels, faceMask, _histogram, 1, &sizes, &range);
ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] * double min, max;
Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8; minMaxLoc(_histogram, &min, &max);
ydest += 2; _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max);
int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex];
int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex];
int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex];
int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8;
*vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128;
*udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128;
}
yline += vpxImage.stride[0] * 2;
vline += vpxImage.stride[1];
uline += vpxImage.stride[2];
}
// encode the frame
vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME);
// start the payload off with the aspect ratio
QByteArray payload(sizeof(float), 0);
*(float*)payload.data() = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height;
// extract the encoded frame
vpx_codec_iter_t iterator = 0;
const vpx_codec_cx_pkt_t* packet;
while ((packet = vpx_codec_get_cx_data(&_colorCodec, &iterator)) != 0) {
if (packet->kind == VPX_CODEC_CX_FRAME_PKT) {
// prepend the length, which will indicate whether there's a depth frame too
payload.append((const char*)&packet->data.frame.sz, sizeof(packet->data.frame.sz));
payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz);
}
}
if (!depth.empty()) {
// warp the face depth without interpolation (because it will contain invalid zero values)
_faceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1);
warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST);
// find the mean of the valid values
qint64 depthTotal = 0;
qint64 depthSamples = 0;
ushort* src = _faceDepth.ptr<ushort>();
const ushort ELEVEN_BIT_MINIMUM = 0;
const ushort ELEVEN_BIT_MAXIMUM = 2047;
for (int i = 0; i < ENCODED_FACE_HEIGHT; i++) {
for (int j = 0; j < ENCODED_FACE_WIDTH; j++) {
ushort depth = *src++;
if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) {
depthTotal += depth;
depthSamples++;
}
} }
} }
float mean = (depthSamples == 0) ? UNINITIALIZED_FACE_DEPTH : depthTotal / (float)depthSamples; RotatedRect faceRect;
if (_searchWindow.area() > 0) {
updateHSVFrame(color, format);
calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range);
bitwise_and(_backProject, _mask, _backProject);
faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1));
Rect faceBounds = faceRect.boundingRect();
Rect imageBounds(0, 0, color.cols, color.rows);
_searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds));
}
const int ENCODED_FACE_WIDTH = 128;
const int ENCODED_FACE_HEIGHT = 128;
encodedWidth = ENCODED_FACE_WIDTH;
encodedHeight = ENCODED_FACE_HEIGHT;
depthBitrateMultiplier = 2;
// smooth the mean over time // correct for 180 degree rotations
const float DEPTH_OFFSET_SMOOTHING = 0.95f; if (faceRect.angle < -90.0f) {
_smoothedMeanFaceDepth = (_smoothedMeanFaceDepth == UNINITIALIZED_FACE_DEPTH) ? mean : faceRect.angle += 180.0f;
glm::mix(mean, _smoothedMeanFaceDepth, DEPTH_OFFSET_SMOOTHING);
} else if (faceRect.angle > 90.0f) {
faceRect.angle -= 180.0f;
}
// compute the smoothed face rect
if (_smoothedFaceRect.size.area() == 0) {
_smoothedFaceRect = faceRect;
} else {
const float FACE_RECT_SMOOTHING = 0.9f;
_smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING);
_smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING);
_smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING);
_smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING);
_smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING);
}
// use the face rect to compute the face transform, aspect ratio
Point2f sourcePoints[4];
_smoothedFaceRect.points(sourcePoints);
Point2f destPoints[] = { Point2f(0, encodedHeight), Point2f(0, 0), Point2f(encodedWidth, 0) };
faceTransform = getAffineTransform(sourcePoints, destPoints);
aspectRatio = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height;
}
const ushort ELEVEN_BIT_MINIMUM = 0;
const uchar EIGHT_BIT_MIDPOINT = 128;
double depthOffset;
if (!depth.empty()) {
if (_videoSendMode == FACE_VIDEO) {
// warp the face depth without interpolation (because it will contain invalid zero values)
_faceDepth.create(encodedHeight, encodedWidth, CV_16UC1);
warpAffine(depth, _faceDepth, faceTransform, _faceDepth.size(), INTER_NEAREST);
} else {
_faceDepth = depth;
}
_smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1);
// smooth the depth over time
const ushort ELEVEN_BIT_MAXIMUM = 2047;
const float DEPTH_SMOOTHING = 0.25f;
ushort* src = _faceDepth.ptr<ushort>();
ushort* dest = _smoothedFaceDepth.ptr<ushort>();
ushort minimumDepth = numeric_limits<ushort>::max();
for (int i = 0; i < encodedHeight; i++) {
for (int j = 0; j < encodedWidth; j++) {
ushort depth = *src++;
if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) {
minimumDepth = min(minimumDepth, depth);
*dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING);
}
dest++;
}
}
const ushort MINIMUM_DEPTH_OFFSET = 64;
const float FIXED_MID_DEPTH = 640.0f;
float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH;
// smooth the mid face depth over time
const float MID_FACE_DEPTH_SMOOTHING = 0.5f;
_smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth :
glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING);
// convert from 11 to 8 bits for preview/local display // convert from 11 to 8 bits for preview/local display
const uchar EIGHT_BIT_MIDPOINT = 128; depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth;
double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMeanFaceDepth;
depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset);
}
// likewise for the encoded representation
QByteArray payload;
if (_videoSendMode != NO_VIDEO) {
if (_colorCodec.name == 0) {
// initialize encoder context(s)
vpx_codec_enc_cfg_t codecConfig;
vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0);
codecConfig.rc_target_bitrate = encodedWidth * encodedHeight *
codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h;
codecConfig.g_w = encodedWidth;
codecConfig.g_h = encodedHeight;
vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0);
if (!depth.empty()) {
codecConfig.rc_target_bitrate *= depthBitrateMultiplier;
vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0);
}
}
Mat transform;
if (_videoSendMode == FACE_VIDEO) {
// resize/rotate face into encoding rectangle
_faceColor.create(encodedHeight, encodedWidth, CV_8UC3);
warpAffine(color, _faceColor, faceTransform, _faceColor.size());
} else {
_faceColor = color;
}
// convert from RGB to YV12: see http://www.fourcc.org/yuv.php and
// http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor
const int ENCODED_BITS_PER_Y = 8;
const int ENCODED_BITS_PER_VU = 2;
const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU;
const int BITS_PER_BYTE = 8;
_encodedFace.resize(encodedWidth * encodedHeight * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE);
vpx_image_t vpxImage;
vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, encodedWidth, encodedHeight, 1,
(unsigned char*)_encodedFace.data());
uchar* yline = vpxImage.planes[0]; uchar* yline = vpxImage.planes[0];
uchar* vline = vpxImage.planes[1]; uchar* vline = vpxImage.planes[1];
uchar* uline = vpxImage.planes[2]; uchar* uline = vpxImage.planes[2];
const uchar EIGHT_BIT_MAXIMUM = 255; const int Y_RED_WEIGHT = (int)(0.299 * 256);
for (int i = 0; i < ENCODED_FACE_HEIGHT; i += 2) { const int Y_GREEN_WEIGHT = (int)(0.587 * 256);
const int Y_BLUE_WEIGHT = (int)(0.114 * 256);
const int V_RED_WEIGHT = (int)(0.713 * 256);
const int U_BLUE_WEIGHT = (int)(0.564 * 256);
int redIndex = 0;
int greenIndex = 1;
int blueIndex = 2;
if (format == GL_BGR) {
redIndex = 2;
blueIndex = 0;
}
for (int i = 0; i < encodedHeight; i += 2) {
uchar* ydest = yline; uchar* ydest = yline;
uchar* vdest = vline; uchar* vdest = vline;
uchar* udest = uline; uchar* udest = uline;
for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { for (int j = 0; j < encodedWidth; j += 2) {
ushort tl = *_faceDepth.ptr<ushort>(i, j); uchar* tl = _faceColor.ptr(i, j);
ushort tr = *_faceDepth.ptr<ushort>(i, j + 1); uchar* tr = _faceColor.ptr(i, j + 1);
ushort bl = *_faceDepth.ptr<ushort>(i + 1, j); uchar* bl = _faceColor.ptr(i + 1, j);
ushort br = *_faceDepth.ptr<ushort>(i + 1, j + 1); uchar* br = _faceColor.ptr(i + 1, j + 1);
uchar mask = EIGHT_BIT_MAXIMUM;
ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : saturate_cast<uchar>(tl + depthOffset); ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8;
ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : saturate_cast<uchar>(tr + depthOffset); ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8;
ydest[vpxImage.stride[0]] = (bl == ELEVEN_BIT_MINIMUM) ? ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] *
(mask = EIGHT_BIT_MIDPOINT) : saturate_cast<uchar>(bl + depthOffset); Y_GREEN_WEIGHT + bl[blueIndex] * Y_BLUE_WEIGHT) >> 8;
ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] *
(mask = EIGHT_BIT_MIDPOINT) : saturate_cast<uchar>(br + depthOffset); Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8;
ydest += 2; ydest += 2;
*vdest++ = mask; int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex];
*udest++ = EIGHT_BIT_MIDPOINT; int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex];
int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex];
int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8;
*vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128;
*udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128;
} }
yline += vpxImage.stride[0] * 2; yline += vpxImage.stride[0] * 2;
vline += vpxImage.stride[1]; vline += vpxImage.stride[1];
@ -681,24 +674,79 @@ void FrameGrabber::grabFrame() {
} }
// encode the frame // encode the frame
vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME); vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME);
// start the payload off with the aspect ratio (zero for full frame)
payload.append((const char*)&aspectRatio, sizeof(float));
// extract the encoded frame // extract the encoded frame
vpx_codec_iter_t iterator = 0; vpx_codec_iter_t iterator = 0;
const vpx_codec_cx_pkt_t* packet; const vpx_codec_cx_pkt_t* packet;
while ((packet = vpx_codec_get_cx_data(&_depthCodec, &iterator)) != 0) { while ((packet = vpx_codec_get_cx_data(&_colorCodec, &iterator)) != 0) {
if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { if (packet->kind == VPX_CODEC_CX_FRAME_PKT) {
// prepend the length, which will indicate whether there's a depth frame too
payload.append((const char*)&packet->data.frame.sz, sizeof(packet->data.frame.sz));
payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz);
} }
} }
if (!depth.empty()) {
// convert with mask
uchar* yline = vpxImage.planes[0];
uchar* vline = vpxImage.planes[1];
uchar* uline = vpxImage.planes[2];
const uchar EIGHT_BIT_MAXIMUM = 255;
for (int i = 0; i < encodedHeight; i += 2) {
uchar* ydest = yline;
uchar* vdest = vline;
uchar* udest = uline;
for (int j = 0; j < encodedWidth; j += 2) {
ushort tl = *_smoothedFaceDepth.ptr<ushort>(i, j);
ushort tr = *_smoothedFaceDepth.ptr<ushort>(i, j + 1);
ushort bl = *_smoothedFaceDepth.ptr<ushort>(i + 1, j);
ushort br = *_smoothedFaceDepth.ptr<ushort>(i + 1, j + 1);
uchar mask = EIGHT_BIT_MAXIMUM;
ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) :
saturate_cast<uchar>(tl + depthOffset);
ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) :
saturate_cast<uchar>(tr + depthOffset);
ydest[vpxImage.stride[0]] = (bl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) :
saturate_cast<uchar>(bl + depthOffset);
ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) :
saturate_cast<uchar>(br + depthOffset);
ydest += 2;
*vdest++ = mask;
*udest++ = EIGHT_BIT_MIDPOINT;
}
yline += vpxImage.stride[0] * 2;
vline += vpxImage.stride[1];
uline += vpxImage.stride[2];
}
// encode the frame
vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME);
// extract the encoded frame
vpx_codec_iter_t iterator = 0;
const vpx_codec_cx_pkt_t* packet;
while ((packet = vpx_codec_get_cx_data(&_depthCodec, &iterator)) != 0) {
if (packet->kind == VPX_CODEC_CX_FRAME_PKT) {
payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz);
}
}
}
} }
QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage",
Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload));
QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame",
Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMeanFaceDepth), Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth),
Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(bool, !payload.isEmpty()),
Q_ARG(JointVector, joints));
} }
bool FrameGrabber::init() { bool FrameGrabber::init() {
@ -767,6 +815,17 @@ void FrameGrabber::updateHSVFrame(const Mat& frame, int format) {
inRange(_hsvFrame, Scalar(0, 55, 65), Scalar(180, 256, 256), _mask); inRange(_hsvFrame, Scalar(0, 55, 65), Scalar(180, 256, 256), _mask);
} }
void FrameGrabber::destroyCodecs() {
if (_colorCodec.name != 0) {
vpx_codec_destroy(&_colorCodec);
_colorCodec.name = 0;
}
if (_depthCodec.name != 0) {
vpx_codec_destroy(&_depthCodec);
_depthCodec.name = 0;
}
}
Joint::Joint(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& projected) : Joint::Joint(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& projected) :
isValid(true), position(position), rotation(rotation), projected(projected) { isValid(true), position(position), rotation(rotation), projected(projected) {
} }

25
interface/src/Webcam.h
View file

@ -44,12 +44,18 @@ public:
Webcam(); Webcam();
~Webcam(); ~Webcam();
FrameGrabber* getGrabber() { return _grabber; }
bool isActive() const { return _active; } bool isActive() const { return _active; }
bool isSending() const { return _sending; }
GLuint getColorTextureID() const { return _colorTextureID; } GLuint getColorTextureID() const { return _colorTextureID; }
GLuint getDepthTextureID() const { return _depthTextureID; } GLuint getDepthTextureID() const { return _depthTextureID; }
const cv::Size2f& getTextureSize() const { return _textureSize; } const cv::Size2f& getTextureSize() const { return _textureSize; }
float getAspectRatio() const { return _aspectRatio; }
const cv::RotatedRect& getFaceRect() const { return _faceRect; } const cv::RotatedRect& getFaceRect() const { return _faceRect; }
const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; } const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; }
@ -57,13 +63,14 @@ public:
const JointVector& getEstimatedJoints() const { return _estimatedJoints; } const JointVector& getEstimatedJoints() const { return _estimatedJoints; }
void reset(); void reset();
void renderPreview(int screenWidth, int screenHeight); void renderPreview(int screenWidth, int screenHeight);
public slots: public slots:
void setEnabled(bool enabled); void setEnabled(bool enabled);
void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float meanFaceDepth, void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth,
const cv::RotatedRect& faceRect, const JointVector& joints); float aspectRatio, const cv::RotatedRect& faceRect, bool sending, const JointVector& joints);
void setSkeletonTrackingOn(bool toggle) { _skeletonTrackingOn = toggle; };
private: private:
@ -72,9 +79,11 @@ private:
bool _enabled; bool _enabled;
bool _active; bool _active;
bool _sending;
GLuint _colorTextureID; GLuint _colorTextureID;
GLuint _depthTextureID; GLuint _depthTextureID;
cv::Size2f _textureSize; cv::Size2f _textureSize;
float _aspectRatio;
cv::RotatedRect _faceRect; cv::RotatedRect _faceRect;
cv::RotatedRect _initialFaceRect; cv::RotatedRect _initialFaceRect;
float _initialFaceDepth; float _initialFaceDepth;
@ -88,6 +97,8 @@ private:
glm::vec3 _estimatedPosition; glm::vec3 _estimatedPosition;
glm::vec3 _estimatedRotation; glm::vec3 _estimatedRotation;
JointVector _estimatedJoints; JointVector _estimatedJoints;
bool _skeletonTrackingOn;
}; };
class FrameGrabber : public QObject { class FrameGrabber : public QObject {
@ -100,16 +111,21 @@ public:
public slots: public slots:
void cycleVideoSendMode();
void reset(); void reset();
void shutdown(); void shutdown();
void grabFrame(); void grabFrame();
private: private:
enum VideoSendMode { NO_VIDEO, FACE_VIDEO, FULL_FRAME_VIDEO, VIDEO_SEND_MODE_COUNT };
bool init(); bool init();
void updateHSVFrame(const cv::Mat& frame, int format); void updateHSVFrame(const cv::Mat& frame, int format);
void destroyCodecs();
bool _initialized; bool _initialized;
VideoSendMode _videoSendMode;
CvCapture* _capture; CvCapture* _capture;
cv::CascadeClassifier _faceCascade; cv::CascadeClassifier _faceCascade;
cv::Mat _hsvFrame; cv::Mat _hsvFrame;
@ -118,13 +134,14 @@ private:
cv::Mat _backProject; cv::Mat _backProject;
cv::Rect _searchWindow; cv::Rect _searchWindow;
cv::Mat _grayDepthFrame; cv::Mat _grayDepthFrame;
float _smoothedMeanFaceDepth; float _smoothedMidFaceDepth;
vpx_codec_ctx_t _colorCodec; vpx_codec_ctx_t _colorCodec;
vpx_codec_ctx_t _depthCodec; vpx_codec_ctx_t _depthCodec;
int _frameCount; int _frameCount;
cv::Mat _faceColor; cv::Mat _faceColor;
cv::Mat _faceDepth; cv::Mat _faceDepth;
cv::Mat _smoothedFaceDepth;
QByteArray _encodedFace; QByteArray _encodedFace;
cv::RotatedRect _smoothedFaceRect; cv::RotatedRect _smoothedFaceRect;

25
interface/src/avatar/Avatar.cpp
View file

@ -315,10 +315,7 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook,
estimatedPosition = webcam->getEstimatedPosition(); estimatedPosition = webcam->getEstimatedPosition();
// apply face data // apply face data
_head.getFace().setColorTextureID(webcam->getColorTextureID()); _head.getFace().setFrameFromWebcam();
_head.getFace().setDepthTextureID(webcam->getDepthTextureID());
_head.getFace().setTextureSize(webcam->getTextureSize());
_head.getFace().setTextureRect(webcam->getFaceRect());
// compute and store the joint rotations // compute and store the joint rotations
const JointVector& joints = webcam->getEstimatedJoints(); const JointVector& joints = webcam->getEstimatedJoints();
@ -335,7 +332,7 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook,
} }
} }
} else { } else {
_head.getFace().setColorTextureID(0); _head.getFace().clearFrame();
} }
_head.setPitch(estimatedRotation.x * amplifyAngle.x + pitchFromTouch); _head.setPitch(estimatedRotation.x * amplifyAngle.x + pitchFromTouch);
_head.setYaw(estimatedRotation.y * amplifyAngle.y + yawFromTouch); _head.setYaw(estimatedRotation.y * amplifyAngle.y + yawFromTouch);
@ -941,21 +938,15 @@ void Avatar::updateCollisionWithSphere(glm::vec3 position, float radius, float d
} }
void Avatar::updateCollisionWithEnvironment(float deltaTime) { void Avatar::updateCollisionWithEnvironment(float deltaTime) {
glm::vec3 up = getBodyUpDirection(); glm::vec3 up = getBodyUpDirection();
float radius = _height * 0.125f; float radius = _height * 0.125f;
const float ENVIRONMENT_SURFACE_ELASTICITY = 1.0f; const float ENVIRONMENT_SURFACE_ELASTICITY = 1.0f;
const float ENVIRONMENT_SURFACE_DAMPING = 0.01; const float ENVIRONMENT_SURFACE_DAMPING = 0.01;
const float ENVIRONMENT_COLLISION_FREQUENCY = 0.05f; const float ENVIRONMENT_COLLISION_FREQUENCY = 0.05f;
const float VISIBLE_GROUND_COLLISION_VELOCITY = 0.2f;
glm::vec3 penetration; glm::vec3 penetration;
if (Application::getInstance()->getEnvironment()->findCapsulePenetration( if (Application::getInstance()->getEnvironment()->findCapsulePenetration(
_position - up * (_pelvisFloatingHeight - radius), _position - up * (_pelvisFloatingHeight - radius),
_position + up * (_height - _pelvisFloatingHeight - radius), radius, penetration)) { _position + up * (_height - _pelvisFloatingHeight - radius), radius, penetration)) {
float velocityTowardCollision = glm::dot(_velocity, glm::normalize(penetration));
if (velocityTowardCollision > VISIBLE_GROUND_COLLISION_VELOCITY) {
Application::getInstance()->setGroundPlaneImpact(1.0f);
}
_lastCollisionPosition = _position; _lastCollisionPosition = _position;
updateCollisionSound(penetration, deltaTime, ENVIRONMENT_COLLISION_FREQUENCY); updateCollisionSound(penetration, deltaTime, ENVIRONMENT_COLLISION_FREQUENCY);
applyHardCollision(penetration, ENVIRONMENT_SURFACE_ELASTICITY, ENVIRONMENT_SURFACE_DAMPING); applyHardCollision(penetration, ENVIRONMENT_SURFACE_ELASTICITY, ENVIRONMENT_SURFACE_DAMPING);
@ -1349,9 +1340,15 @@ float Avatar::getBallRenderAlpha(int ball, bool lookingInMirror) const {
} }
void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) {
// Render the body as balls and cones if (_head.getFace().isFullFrame()) {
if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) { // Render the full-frame video
float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror);
if (alpha > 0.0f) {
_head.getFace().render(1.0f);
}
} else if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) {
// Render the body as balls and cones
for (int b = 0; b < NUM_AVATAR_BODY_BALLS; b++) { for (int b = 0; b < NUM_AVATAR_BODY_BALLS; b++) {
float alpha = getBallRenderAlpha(b, lookingInMirror); float alpha = getBallRenderAlpha(b, lookingInMirror);

319
interface/src/avatar/Face.cpp
View file

@ -17,6 +17,7 @@
#include "Avatar.h" #include "Avatar.h"
#include "Head.h" #include "Head.h"
#include "Face.h" #include "Face.h"
#include "Webcam.h"
#include "renderer/ProgramObject.h" #include "renderer/ProgramObject.h"
using namespace cv; using namespace cv;
@ -25,7 +26,6 @@ ProgramObject* Face::_program = 0;
int Face::_texCoordCornerLocation; int Face::_texCoordCornerLocation;
int Face::_texCoordRightLocation; int Face::_texCoordRightLocation;
int Face::_texCoordUpLocation; int Face::_texCoordUpLocation;
int Face::_aspectRatioLocation;
GLuint Face::_vboID; GLuint Face::_vboID;
GLuint Face::_iboID; GLuint Face::_iboID;
@ -55,17 +55,25 @@ Face::~Face() {
} }
} }
void Face::setTextureRect(const cv::RotatedRect& textureRect) { void Face::setFrameFromWebcam() {
_textureRect = textureRect; Webcam* webcam = Application::getInstance()->getWebcam();
_aspectRatio = _textureRect.size.width / _textureRect.size.height; if (webcam->isSending()) {
_colorTextureID = webcam->getColorTextureID();
_depthTextureID = webcam->getDepthTextureID();
_textureSize = webcam->getTextureSize();
_textureRect = webcam->getFaceRect();
_aspectRatio = webcam->getAspectRatio();
} else {
clearFrame();
}
} }
void Face::clearFrame() {
_colorTextureID = 0;
}
int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) {
if (_colorCodec.name == 0) {
// initialize decoder context
vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0);
}
// skip the header
unsigned char* packetPosition = packetData; unsigned char* packetPosition = packetData;
int frameCount = *(uint32_t*)packetPosition; int frameCount = *(uint32_t*)packetPosition;
@ -89,110 +97,135 @@ int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) {
int payloadSize = dataBytes - (packetPosition - packetData); int payloadSize = dataBytes - (packetPosition - packetData);
memcpy(_arrivingFrame.data() + frameOffset, packetPosition, payloadSize); memcpy(_arrivingFrame.data() + frameOffset, packetPosition, payloadSize);
if ((_frameBytesRemaining -= payloadSize) <= 0) { if ((_frameBytesRemaining -= payloadSize) > 0) {
float aspectRatio = *(const float*)_arrivingFrame.constData(); return dataBytes; // wait for the rest of the frame
size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float)); }
const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t));
vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0); if (frameSize == 0) {
vpx_codec_iter_t iterator = 0; // destroy the codecs, if we have any
vpx_image_t* image; destroyCodecs();
while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) {
// convert from YV12 to RGB // disables video data
Mat color(image->d_h, image->d_w, CV_8UC3); QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, Mat()),
uchar* yline = image->planes[0]; Q_ARG(cv::Mat, Mat()), Q_ARG(float, 0.0f));
uchar* vline = image->planes[1]; return dataBytes;
uchar* uline = image->planes[2]; }
const int RED_V_WEIGHT = (int)(1.403 * 256);
const int GREEN_V_WEIGHT = (int)(0.714 * 256); // the switch from full frame to not (or vice versa) requires us to reinit the codecs
const int GREEN_U_WEIGHT = (int)(0.344 * 256); float aspectRatio = *(const float*)_arrivingFrame.constData();
const int BLUE_U_WEIGHT = (int)(1.773 * 256); bool fullFrame = (aspectRatio == FULL_FRAME_ASPECT);
for (int i = 0; i < image->d_h; i += 2) { if (fullFrame != _lastFullFrame) {
uchar* ysrc = yline; destroyCodecs();
uchar* vsrc = vline; _lastFullFrame = fullFrame;
uchar* usrc = uline; }
for (int j = 0; j < image->d_w; j += 2) {
uchar* tl = color.ptr(i, j); if (_colorCodec.name == 0) {
uchar* tr = color.ptr(i, j + 1); // initialize decoder context
uchar* bl = color.ptr(i + 1, j); vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0);
uchar* br = color.ptr(i + 1, j + 1); }
int v = *vsrc++ - 128; size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float));
int u = *usrc++ - 128; const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t));
vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0);
int redOffset = (RED_V_WEIGHT * v) >> 8; vpx_codec_iter_t iterator = 0;
int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; vpx_image_t* image;
int blueOffset = (BLUE_U_WEIGHT * u) >> 8; while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) {
// convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and
int ytl = ysrc[0]; // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor
int ytr = ysrc[1]; Mat color(image->d_h, image->d_w, CV_8UC3);
int ybl = ysrc[image->w]; uchar* yline = image->planes[0];
int ybr = ysrc[image->w + 1]; uchar* vline = image->planes[1];
ysrc += 2; uchar* uline = image->planes[2];
const int RED_V_WEIGHT = (int)(1.403 * 256);
tl[0] = ytl + redOffset; const int GREEN_V_WEIGHT = (int)(0.714 * 256);
tl[1] = ytl - greenOffset; const int GREEN_U_WEIGHT = (int)(0.344 * 256);
tl[2] = ytl + blueOffset; const int BLUE_U_WEIGHT = (int)(1.773 * 256);
for (int i = 0; i < image->d_h; i += 2) {
tr[0] = ytr + redOffset; uchar* ysrc = yline;
tr[1] = ytr - greenOffset; uchar* vsrc = vline;
tr[2] = ytr + blueOffset; uchar* usrc = uline;
for (int j = 0; j < image->d_w; j += 2) {
bl[0] = ybl + redOffset; uchar* tl = color.ptr(i, j);
bl[1] = ybl - greenOffset; uchar* tr = color.ptr(i, j + 1);
bl[2] = ybl + blueOffset; uchar* bl = color.ptr(i + 1, j);
uchar* br = color.ptr(i + 1, j + 1);
br[0] = ybr + redOffset;
br[1] = ybr - greenOffset; int v = *vsrc++ - 128;
br[2] = ybr + blueOffset; int u = *usrc++ - 128;
}
yline += image->stride[0] * 2; int redOffset = (RED_V_WEIGHT * v) >> 8;
vline += image->stride[1]; int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8;
uline += image->stride[2]; int blueOffset = (BLUE_U_WEIGHT * u) >> 8;
int ytl = ysrc[0];
int ytr = ysrc[1];
int ybl = ysrc[image->w];
int ybr = ysrc[image->w + 1];
ysrc += 2;
tl[0] = ytl + redOffset;
tl[1] = ytl - greenOffset;
tl[2] = ytl + blueOffset;
tr[0] = ytr + redOffset;
tr[1] = ytr - greenOffset;
tr[2] = ytr + blueOffset;
bl[0] = ybl + redOffset;
bl[1] = ybl - greenOffset;
bl[2] = ybl + blueOffset;
br[0] = ybr + redOffset;
br[1] = ybr - greenOffset;
br[2] = ybr + blueOffset;
} }
Mat depth; yline += image->stride[0] * 2;
vline += image->stride[1];
const uint8_t* depthData = colorData + colorSize; uline += image->stride[2];
int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData());
if (depthSize > 0) {
if (_depthCodec.name == 0) {
// initialize decoder context
vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0);
}
vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0);
vpx_codec_iter_t iterator = 0;
vpx_image_t* image;
while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) {
depth.create(image->d_h, image->d_w, CV_8UC1);
uchar* yline = image->planes[0];
uchar* vline = image->planes[1];
const uchar EIGHT_BIT_MAXIMUM = 255;
const uchar MASK_THRESHOLD = 192;
for (int i = 0; i < image->d_h; i += 2) {
uchar* ysrc = yline;
uchar* vsrc = vline;
for (int j = 0; j < image->d_w; j += 2) {
if (*vsrc++ < MASK_THRESHOLD) {
*depth.ptr(i, j) = EIGHT_BIT_MAXIMUM;
*depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM;
*depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM;
*depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM;
} else {
*depth.ptr(i, j) = ysrc[0];
*depth.ptr(i, j + 1) = ysrc[1];
*depth.ptr(i + 1, j) = ysrc[image->stride[0]];
*depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1];
}
ysrc += 2;
}
yline += image->stride[0] * 2;
vline += image->stride[1];
}
}
}
QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color),
Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio));
} }
Mat depth;
const uint8_t* depthData = colorData + colorSize;
int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData());
if (depthSize > 0) {
if (_depthCodec.name == 0) {
// initialize decoder context
vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0);
}
vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0);
vpx_codec_iter_t iterator = 0;
vpx_image_t* image;
while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) {
depth.create(image->d_h, image->d_w, CV_8UC1);
uchar* yline = image->planes[0];
uchar* vline = image->planes[1];
const uchar EIGHT_BIT_MAXIMUM = 255;
const uchar MASK_THRESHOLD = 192;
for (int i = 0; i < image->d_h; i += 2) {
uchar* ysrc = yline;
uchar* vsrc = vline;
for (int j = 0; j < image->d_w; j += 2) {
if (*vsrc++ < MASK_THRESHOLD) {
*depth.ptr(i, j) = EIGHT_BIT_MAXIMUM;
*depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM;
*depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM;
*depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM;
} else {
*depth.ptr(i, j) = ysrc[0];
*depth.ptr(i, j + 1) = ysrc[1];
*depth.ptr(i + 1, j) = ysrc[image->stride[0]];
*depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1];
}
ysrc += 2;
}
yline += image->stride[0] * 2;
vline += image->stride[1];
}
}
}
QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color),
Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio));
} }
return dataBytes; return dataBytes;
@ -208,9 +241,22 @@ bool Face::render(float alpha) {
glm::quat orientation = _owningHead->getOrientation(); glm::quat orientation = _owningHead->getOrientation();
glm::vec3 axis = glm::axis(orientation); glm::vec3 axis = glm::axis(orientation);
glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z);
float scale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale();
glScalef(scale, scale, scale); float aspect, xScale, zScale;
if (_aspectRatio == FULL_FRAME_ASPECT) {
aspect = _textureSize.width / _textureSize.height;
const float FULL_FRAME_SCALE = 0.5f;
xScale = FULL_FRAME_SCALE * _owningHead->getScale();
zScale = xScale * 0.3f;
} else {
aspect = _aspectRatio;
xScale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale();
zScale = xScale * 1.5f;
glTranslatef(0.0f, -xScale * 0.75f, -xScale);
}
glScalef(xScale, xScale / aspect, zScale);
glColor4f(1.0f, 1.0f, 1.0f, alpha); glColor4f(1.0f, 1.0f, 1.0f, alpha);
Point2f points[4]; Point2f points[4];
@ -243,7 +289,6 @@ bool Face::render(float alpha) {
_texCoordCornerLocation = _program->uniformLocation("texCoordCorner"); _texCoordCornerLocation = _program->uniformLocation("texCoordCorner");
_texCoordRightLocation = _program->uniformLocation("texCoordRight"); _texCoordRightLocation = _program->uniformLocation("texCoordRight");
_texCoordUpLocation = _program->uniformLocation("texCoordUp"); _texCoordUpLocation = _program->uniformLocation("texCoordUp");
_aspectRatioLocation = _program->uniformLocation("aspectRatio");
glGenBuffers(1, &_vboID); glGenBuffers(1, &_vboID);
glBindBuffer(GL_ARRAY_BUFFER, _vboID); glBindBuffer(GL_ARRAY_BUFFER, _vboID);
@ -292,7 +337,6 @@ bool Face::render(float alpha) {
(points[3].x - points[0].x) / _textureSize.width, (points[3].y - points[0].y) / _textureSize.height); (points[3].x - points[0].x) / _textureSize.width, (points[3].y - points[0].y) / _textureSize.height);
_program->setUniformValue(_texCoordUpLocation, _program->setUniformValue(_texCoordUpLocation,
(points[1].x - points[0].x) / _textureSize.width, (points[1].y - points[0].y) / _textureSize.height); (points[1].x - points[0].x) / _textureSize.width, (points[1].y - points[0].y) / _textureSize.height);
_program->setUniformValue(_aspectRatioLocation, _aspectRatio);
glEnableClientState(GL_VERTEX_ARRAY); glEnableClientState(GL_VERTEX_ARRAY);
glVertexPointer(2, GL_FLOAT, 0, 0); glVertexPointer(2, GL_FLOAT, 0, 0);
@ -324,13 +368,13 @@ bool Face::render(float alpha) {
glBegin(GL_QUADS); glBegin(GL_QUADS);
glTexCoord2f(points[0].x / _textureSize.width, points[0].y / _textureSize.height); glTexCoord2f(points[0].x / _textureSize.width, points[0].y / _textureSize.height);
glVertex3f(0.5f, -0.5f / _aspectRatio, -0.5f); glVertex3f(0.5f, -0.5f, 0.0f);
glTexCoord2f(points[1].x / _textureSize.width, points[1].y / _textureSize.height); glTexCoord2f(points[1].x / _textureSize.width, points[1].y / _textureSize.height);
glVertex3f(0.5f, 0.5f / _aspectRatio, -0.5f); glVertex3f(0.5f, 0.5f, 0.0f);
glTexCoord2f(points[2].x / _textureSize.width, points[2].y / _textureSize.height); glTexCoord2f(points[2].x / _textureSize.width, points[2].y / _textureSize.height);
glVertex3f(-0.5f, 0.5f / _aspectRatio, -0.5f); glVertex3f(-0.5f, 0.5f, 0.0f);
glTexCoord2f(points[3].x / _textureSize.width, points[3].y / _textureSize.height); glTexCoord2f(points[3].x / _textureSize.width, points[3].y / _textureSize.height);
glVertex3f(-0.5f, -0.5f / _aspectRatio, -0.5f); glVertex3f(-0.5f, -0.5f, 0.0f);
glEnd(); glEnd();
glDisable(GL_TEXTURE_2D); glDisable(GL_TEXTURE_2D);
@ -348,23 +392,40 @@ void Face::cycleRenderMode() {
} }
void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) { void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) {
if (color.empty()) {
// release our textures, if any; there's no more video
if (_colorTextureID != 0) {
glDeleteTextures(1, &_colorTextureID);
_colorTextureID = 0;
}
if (_depthTextureID != 0) {
glDeleteTextures(1, &_depthTextureID);
_depthTextureID = 0;
}
return;
}
if (_colorTextureID == 0) { if (_colorTextureID == 0) {
glGenTextures(1, &_colorTextureID); glGenTextures(1, &_colorTextureID);
glBindTexture(GL_TEXTURE_2D, _colorTextureID); }
glBindTexture(GL_TEXTURE_2D, _colorTextureID);
bool recreateTextures = (_textureSize.width != color.cols || _textureSize.height != color.rows);
if (recreateTextures) {
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, color.cols, color.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, color.cols, color.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, color.ptr());
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
_textureSize = color.size(); _textureSize = color.size();
_textureRect = RotatedRect(Point2f(color.cols * 0.5f, color.rows * 0.5f), _textureSize, 0.0f); _textureRect = RotatedRect(Point2f(color.cols * 0.5f, color.rows * 0.5f), _textureSize, 0.0f);
} else { } else {
glBindTexture(GL_TEXTURE_2D, _colorTextureID);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, color.cols, color.rows, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, color.cols, color.rows, GL_RGB, GL_UNSIGNED_BYTE, color.ptr());
} }
if (!depth.empty()) { if (!depth.empty()) {
if (_depthTextureID == 0) { if (_depthTextureID == 0) {
glGenTextures(1, &_depthTextureID); glGenTextures(1, &_depthTextureID);
glBindTexture(GL_TEXTURE_2D, _depthTextureID); }
glBindTexture(GL_TEXTURE_2D, _depthTextureID);
if (recreateTextures) {
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depth.cols, depth.rows, 0, glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depth.cols, depth.rows, 0,
GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr()); GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr());
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
@ -380,3 +441,13 @@ void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRati
_aspectRatio = aspectRatio; _aspectRatio = aspectRatio;
} }
void Face::destroyCodecs() {
if (_colorCodec.name != 0) {
vpx_codec_destroy(&_colorCodec);
_colorCodec.name = 0;
}
if (_depthCodec.name != 0) {
vpx_codec_destroy(&_depthCodec);
_depthCodec.name = 0;
}
}

14
interface/src/avatar/Face.h
View file

@ -20,6 +20,8 @@
class Head; class Head;
class ProgramObject; class ProgramObject;
const float FULL_FRAME_ASPECT = 0.0f;
class Face : public QObject { class Face : public QObject {
Q_OBJECT Q_OBJECT
@ -28,10 +30,10 @@ public:
Face(Head* owningHead); Face(Head* owningHead);
~Face(); ~Face();
void setColorTextureID(GLuint colorTextureID) { _colorTextureID = colorTextureID; } bool isFullFrame() const { return _colorTextureID != 0 && _aspectRatio == FULL_FRAME_ASPECT; }
void setDepthTextureID(GLuint depthTextureID) { _depthTextureID = depthTextureID; }
void setTextureSize(const cv::Size2f& textureSize) { _textureSize = textureSize; } void setFrameFromWebcam();
void setTextureRect(const cv::RotatedRect& textureRect); void clearFrame();
int processVideoMessage(unsigned char* packetData, size_t dataBytes); int processVideoMessage(unsigned char* packetData, size_t dataBytes);
@ -49,6 +51,8 @@ private:
enum RenderMode { MESH, POINTS, RENDER_MODE_COUNT }; enum RenderMode { MESH, POINTS, RENDER_MODE_COUNT };
void destroyCodecs();
Head* _owningHead; Head* _owningHead;
RenderMode _renderMode; RenderMode _renderMode;
GLuint _colorTextureID; GLuint _colorTextureID;
@ -59,6 +63,7 @@ private:
vpx_codec_ctx_t _colorCodec; vpx_codec_ctx_t _colorCodec;
vpx_codec_ctx_t _depthCodec; vpx_codec_ctx_t _depthCodec;
bool _lastFullFrame;
QByteArray _arrivingFrame; QByteArray _arrivingFrame;
int _frameCount; int _frameCount;
@ -68,7 +73,6 @@ private:
static int _texCoordCornerLocation; static int _texCoordCornerLocation;
static int _texCoordRightLocation; static int _texCoordRightLocation;
static int _texCoordUpLocation; static int _texCoordUpLocation;
static int _aspectRatioLocation;
static GLuint _vboID; static GLuint _vboID;
static GLuint _iboID; static GLuint _iboID;
}; };

209
interface/src/avatar/Hand.cpp
View file

@ -51,6 +51,7 @@ void Hand::reset() {
void Hand::simulate(float deltaTime, bool isMine) { void Hand::simulate(float deltaTime, bool isMine) {
if (_isRaveGloveActive) { if (_isRaveGloveActive) {
updateRaveGloveParticles(deltaTime); updateRaveGloveParticles(deltaTime);
} }
@ -63,7 +64,8 @@ void Hand::calculateGeometry() {
_basePosition = head.getPosition() + head.getOrientation() * offset; _basePosition = head.getPosition() + head.getOrientation() * offset;
_baseOrientation = head.getOrientation(); _baseOrientation = head.getOrientation();
_leapBalls.clear(); // generate finger tip balls....
_leapFingerTipBalls.clear();
for (size_t i = 0; i < getNumPalms(); ++i) { for (size_t i = 0; i < getNumPalms(); ++i) {
PalmData& palm = getPalms()[i]; PalmData& palm = getPalms()[i];
if (palm.isActive()) { if (palm.isActive()) {
@ -71,8 +73,8 @@ void Hand::calculateGeometry() {
FingerData& finger = palm.getFingers()[f]; FingerData& finger = palm.getFingers()[f];
if (finger.isActive()) { if (finger.isActive()) {
const float standardBallRadius = 0.01f; const float standardBallRadius = 0.01f;
_leapBalls.resize(_leapBalls.size() + 1); _leapFingerTipBalls.resize(_leapFingerTipBalls.size() + 1);
HandBall& ball = _leapBalls.back(); HandBall& ball = _leapFingerTipBalls.back();
ball.rotation = _baseOrientation; ball.rotation = _baseOrientation;
ball.position = finger.getTipPosition(); ball.position = finger.getTipPosition();
ball.radius = standardBallRadius; ball.radius = standardBallRadius;
@ -82,6 +84,27 @@ void Hand::calculateGeometry() {
} }
} }
} }
// generate finger rot balls....
_leapFingerRootBalls.clear();
for (size_t i = 0; i < getNumPalms(); ++i) {
PalmData& palm = getPalms()[i];
if (palm.isActive()) {
for (size_t f = 0; f < palm.getNumFingers(); ++f) {
FingerData& finger = palm.getFingers()[f];
if (finger.isActive()) {
const float standardBallRadius = 0.01f;
_leapFingerRootBalls.resize(_leapFingerRootBalls.size() + 1);
HandBall& ball = _leapFingerRootBalls.back();
ball.rotation = _baseOrientation;
ball.position = finger.getRootPosition();
ball.radius = standardBallRadius;
ball.touchForce = 0.0;
ball.isCollidable = true;
}
}
}
}
} }
void Hand::setRaveGloveEffectsMode(QKeyEvent* event) { void Hand::setRaveGloveEffectsMode(QKeyEvent* event) {
@ -120,8 +143,9 @@ void Hand::render(bool lookingInMirror) {
glEnable(GL_RESCALE_NORMAL); glEnable(GL_RESCALE_NORMAL);
if ( SHOW_LEAP_HAND ) { if ( SHOW_LEAP_HAND ) {
renderFingerTrails(); //renderLeapHands();
renderHandSpheres(); renderLeapFingerTrails();
renderLeapHandSpheres();
} }
} }
@ -153,18 +177,64 @@ void Hand::renderRaveGloveStage() {
} }
} }
void Hand::renderHandSpheres() {
void Hand::renderLeapHands() {
for (size_t i = 0; i < getNumPalms(); ++i) {
PalmData& hand = getPalms()[i];
if (hand.isActive()) {
renderLeapHand(hand);
}
}
}
void Hand::renderLeapHand(PalmData& hand) {
glPushMatrix();
const float palmThickness = 0.002f;
glColor4f(0.5f, 0.5f, 0.5f, 1.0);
glm::vec3 tip = hand.getPosition();
glm::vec3 root = hand.getPosition() + hand.getNormal() * palmThickness;
Avatar::renderJointConnectingCone(root, tip, 0.05, 0.03);
for (size_t f = 0; f < hand.getNumFingers(); ++f) {
FingerData& finger = hand.getFingers()[f];
if (finger.isActive()) {
glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, 0.5);
glm::vec3 tip = finger.getTipPosition();
glm::vec3 root = finger.getRootPosition();
Avatar::renderJointConnectingCone(root, tip, 0.001, 0.003);
}
}
glPopMatrix();
}
void Hand::renderLeapHandSpheres() {
glPushMatrix(); glPushMatrix();
// Draw the leap balls // Draw the leap balls
for (size_t i = 0; i < _leapBalls.size(); i++) { for (size_t i = 0; i < _leapFingerTipBalls.size(); i++) {
float alpha = 1.0f; float alpha = 1.0f;
if (alpha > 0.0f) { if (alpha > 0.0f) {
glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, alpha); glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, alpha);
glPushMatrix(); glPushMatrix();
glTranslatef(_leapBalls[i].position.x, _leapBalls[i].position.y, _leapBalls[i].position.z); glTranslatef(_leapFingerTipBalls[i].position.x, _leapFingerTipBalls[i].position.y, _leapFingerTipBalls[i].position.z);
glutSolidSphere(_leapBalls[i].radius, 20.0f, 20.0f); glutSolidSphere(_leapFingerTipBalls[i].radius, 20.0f, 20.0f);
glPopMatrix();
}
}
for (size_t i = 0; i < _leapFingerRootBalls.size(); i++) {
float alpha = 1.0f;
if (alpha > 0.0f) {
glColor4f(0.3f, 0.4f, 0.6f, alpha);
glPushMatrix();
glTranslatef(_leapFingerRootBalls[i].position.x, _leapFingerRootBalls[i].position.y, _leapFingerRootBalls[i].position.z);
glutSolidSphere(_leapFingerRootBalls[i].radius, 20.0f, 20.0f);
glPopMatrix(); glPopMatrix();
} }
} }
@ -200,7 +270,7 @@ void Hand::renderHandSpheres() {
glPopMatrix(); glPopMatrix();
} }
void Hand::renderFingerTrails() { void Hand::renderLeapFingerTrails() {
// Draw the finger root cones // Draw the finger root cones
for (size_t i = 0; i < getNumPalms(); ++i) { for (size_t i = 0; i < getNumPalms(); ++i) {
PalmData& palm = getPalms()[i]; PalmData& palm = getPalms()[i];
@ -229,6 +299,7 @@ void Hand::renderFingerTrails() {
} }
} }
void Hand::setLeapHands(const std::vector<glm::vec3>& handPositions, void Hand::setLeapHands(const std::vector<glm::vec3>& handPositions,
const std::vector<glm::vec3>& handNormals) { const std::vector<glm::vec3>& handNormals) {
for (size_t i = 0; i < getNumPalms(); ++i) { for (size_t i = 0; i < getNumPalms(); ++i) {
@ -244,69 +315,28 @@ void Hand::setLeapHands(const std::vector<glm::vec3>& handPositions,
} }
} }
// call this right after the geometry of the leap hands are set
// call this soon after the geometry of the leap hands are set
void Hand::updateRaveGloveEmitters() { void Hand::updateRaveGloveEmitters() {
bool debug = false; for (size_t i = 0; i < NUM_FINGERS; i++) {
_raveGloveParticleSystem.setEmitterActive(_raveGloveEmitter[i], false);
}
if (_raveGloveInitialized) { for (size_t i = 0; i < _leapFingerTipBalls.size(); i++) {
if (i < NUM_FINGERS) {
if(debug) printf( "\n" ); glm::vec3 fingerDirection = _leapFingerTipBalls[i].position - _leapFingerRootBalls[i].position;
if(debug) printf( "------------------------------------\n" ); float fingerLength = glm::length(fingerDirection);
if(debug) printf( "updating rave glove emitters:\n" );
if(debug) printf( "------------------------------------\n" );
int emitterIndex = 0;
for (size_t i = 0; i < getNumPalms(); ++i) {
PalmData& palm = getPalms()[i];
if(debug) printf( "\n" ); if (fingerLength > 0.0f) {
if(debug) printf( "palm %d ", (int)i ); fingerDirection /= fingerLength;
if (palm.isActive()) {
if(debug) printf( "is active\n" );
for (size_t f = 0; f < palm.getNumFingers(); ++f) {
FingerData& finger = palm.getFingers()[f];
if(debug) printf( "emitterIndex %d: ", emitterIndex );
if (finger.isActive()) {
if ((emitterIndex >=0)
&& (emitterIndex < NUM_FINGERS)) {
assert(emitterIndex >=0 );
assert(emitterIndex < NUM_FINGERS );
if(debug) printf( "_raveGloveEmitter[%d] = %d\n", emitterIndex, _raveGloveEmitter[emitterIndex] );
glm::vec3 fingerDirection = finger.getTipPosition() - finger.getRootPosition();
float fingerLength = glm::length(fingerDirection);
if (fingerLength > 0.0f) {
fingerDirection /= fingerLength;
} else {
fingerDirection = IDENTITY_UP;
}
assert(_raveGloveEmitter[emitterIndex] >=0 );
assert(_raveGloveEmitter[emitterIndex] < NUM_FINGERS );
_raveGloveParticleSystem.setEmitterPosition (_raveGloveEmitter[emitterIndex], finger.getTipPosition());
_raveGloveParticleSystem.setEmitterDirection(_raveGloveEmitter[emitterIndex], fingerDirection);
}
} else {
if(debug) printf( "BOGUS finger\n" );
}
emitterIndex ++;
}
} else { } else {
if(debug) printf( "is NOT active\n" ); fingerDirection = IDENTITY_UP;
} }
_raveGloveParticleSystem.setEmitterActive (_raveGloveEmitter[i], true);
_raveGloveParticleSystem.setEmitterPosition (_raveGloveEmitter[i], _leapFingerTipBalls[i].position);
_raveGloveParticleSystem.setEmitterDirection(_raveGloveEmitter[i], fingerDirection);
} }
} }
} }
@ -317,16 +347,11 @@ void Hand::updateRaveGloveParticles(float deltaTime) {
if (!_raveGloveInitialized) { if (!_raveGloveInitialized) {
//printf( "Initializing rave glove emitters:\n" );
//printf( "The indices of the emitters are:\n" );
// start up the rave glove finger particles... // start up the rave glove finger particles...
for ( int f = 0; f< NUM_FINGERS; f ++ ) { for ( int f = 0; f< NUM_FINGERS; f ++ ) {
_raveGloveEmitter[f] = _raveGloveParticleSystem.addEmitter(); _raveGloveEmitter[f] = _raveGloveParticleSystem.addEmitter();
assert( _raveGloveEmitter[f] >= 0 ); assert( _raveGloveEmitter[f] >= 0 );
assert( _raveGloveEmitter[f] != NULL_EMITTER ); assert( _raveGloveEmitter[f] != NULL_EMITTER );
//printf( "%d\n", _raveGloveEmitter[f] );
} }
setRaveGloveMode(RAVE_GLOVE_EFFECTS_MODE_FIRE); setRaveGloveMode(RAVE_GLOVE_EFFECTS_MODE_FIRE);
@ -339,13 +364,13 @@ void Hand::updateRaveGloveParticles(float deltaTime) {
// this rave glove effect oscillates though various colors and radii that are meant to show off some effects // this rave glove effect oscillates though various colors and radii that are meant to show off some effects
if (_raveGloveMode == RAVE_GLOVE_EFFECTS_MODE_THROBBING_COLOR) { if (_raveGloveMode == RAVE_GLOVE_EFFECTS_MODE_THROBBING_COLOR) {
ParticleSystem::ParticleAttributes attributes; ParticleSystem::ParticleAttributes attributes;
float red = 0.5f + 0.5f * sinf(_raveGloveClock * 1.4f); float red = 0.5f + 0.5f * sinf(_raveGloveClock * 2.4f);
float green = 0.5f + 0.5f * cosf(_raveGloveClock * 1.7f); float green = 0.5f + 0.5f * cosf(_raveGloveClock * 2.7f);
float blue = 0.5f + 0.5f * sinf(_raveGloveClock * 2.0f); float blue = 0.5f + 0.5f * sinf(_raveGloveClock * 3.0f);
float alpha = 1.0f; float alpha = 1.0f;
attributes.color = glm::vec4(red, green, blue, alpha); attributes.color = glm::vec4(red, green, blue, alpha);
attributes.radius = 0.01f + 0.005f * sinf(_raveGloveClock * 2.2f); attributes.radius = 0.01f + 0.003f * sinf(_raveGloveClock * 50.0f);
attributes.modulationAmplitude = 0.0f; attributes.modulationAmplitude = 0.0f;
for ( int f = 0; f< NUM_FINGERS; f ++ ) { for ( int f = 0; f< NUM_FINGERS; f ++ ) {
@ -360,6 +385,8 @@ void Hand::updateRaveGloveParticles(float deltaTime) {
} }
} }
void Hand::setRaveGloveMode(int mode) { void Hand::setRaveGloveMode(int mode) {
_raveGloveMode = mode; _raveGloveMode = mode;
@ -376,7 +403,7 @@ void Hand::setRaveGloveMode(int mode) {
if (mode == RAVE_GLOVE_EFFECTS_MODE_THROBBING_COLOR) { if (mode == RAVE_GLOVE_EFFECTS_MODE_THROBBING_COLOR) {
_raveGloveParticleSystem.setParticleRenderStyle (_raveGloveEmitter[f], PARTICLE_RENDER_STYLE_SPHERE ); _raveGloveParticleSystem.setParticleRenderStyle (_raveGloveEmitter[f], PARTICLE_RENDER_STYLE_SPHERE );
_raveGloveParticleSystem.setShowingEmitterBaseParticle(_raveGloveEmitter[f], true ); _raveGloveParticleSystem.setShowingEmitterBaseParticle(_raveGloveEmitter[f], true );
_raveGloveParticleSystem.setEmitterParticleLifespan (_raveGloveEmitter[f], 0.0f ); _raveGloveParticleSystem.setEmitterParticleLifespan (_raveGloveEmitter[f], 0.03f );
_raveGloveParticleSystem.setEmitterThrust (_raveGloveEmitter[f], 0.0f ); _raveGloveParticleSystem.setEmitterThrust (_raveGloveEmitter[f], 0.0f );
_raveGloveParticleSystem.setEmitterRate (_raveGloveEmitter[f], 30.0f ); _raveGloveParticleSystem.setEmitterRate (_raveGloveEmitter[f], 30.0f );
_raveGloveParticleSystem.setEmitterParticleResolution (_raveGloveEmitter[f], 20 ); _raveGloveParticleSystem.setEmitterParticleResolution (_raveGloveEmitter[f], 20 );
@ -650,7 +677,7 @@ void Hand::setRaveGloveMode(int mode) {
_raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes); _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes);
//----------------------------------------- //-----------------------------------------
// throb // long sparkler
//----------------------------------------- //-----------------------------------------
} else if (mode == RAVE_GLOVE_EFFECTS_MODE_LONG_SPARKLER) { } else if (mode == RAVE_GLOVE_EFFECTS_MODE_LONG_SPARKLER) {
@ -672,6 +699,30 @@ void Hand::setRaveGloveMode(int mode) {
_raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_1, attributes); _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_1, attributes);
_raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_2, attributes); _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_2, attributes);
_raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes); _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes);
//-----------------------------------------
// throb
//-----------------------------------------
} else if (mode == RAVE_GLOVE_EFFECTS_MODE_THROB) {
_raveGloveParticleSystem.setParticleRenderStyle (_raveGloveEmitter[f], PARTICLE_RENDER_STYLE_SPHERE );
_raveGloveParticleSystem.setShowingEmitterBaseParticle(_raveGloveEmitter[f], true );
_raveGloveParticleSystem.setEmitterParticleLifespan (_raveGloveEmitter[f], 0.03 );
_raveGloveParticleSystem.setEmitterThrust (_raveGloveEmitter[f], 0.0f );
_raveGloveParticleSystem.setEmitterRate (_raveGloveEmitter[f], 30.0 );
_raveGloveParticleSystem.setEmitterParticleResolution (_raveGloveEmitter[f], 20 );
_raveGloveParticleSystem.setParticleAttributesToDefault(&attributes);
attributes.radius = 0.01f;
attributes.color = glm::vec4( 0.1f, 0.2f, 0.4f, 0.5f);
attributes.modulationAmplitude = 0.5;
attributes.modulationRate = 3.0;
attributes.modulationStyle = COLOR_MODULATION_STYLE_LIGHTNESS_WAVE;
_raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_0, attributes);
_raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_1, attributes);
_raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_2, attributes);
_raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes);
} }
} }
} }

14
interface/src/avatar/Hand.h
View file

@ -65,8 +65,9 @@ public:
void setRaveGloveEffectsMode(QKeyEvent* event); void setRaveGloveEffectsMode(QKeyEvent* event);
// getters // getters
const glm::vec3& getLeapBallPosition (int ball) const { return _leapBalls[ball].position;} const glm::vec3& getLeapFingerTipBallPosition (int ball) const { return _leapFingerTipBalls [ball].position;}
bool isRaveGloveActive () const { return _isRaveGloveActive; } const glm::vec3& getLeapFingerRootBallPosition(int ball) const { return _leapFingerRootBalls[ball].position;}
bool isRaveGloveActive() const { return _isRaveGloveActive; }
private: private:
// disallow copies of the Hand, copy of owning Avatar is disallowed too // disallow copies of the Hand, copy of owning Avatar is disallowed too
@ -84,7 +85,8 @@ private:
float _renderAlpha; float _renderAlpha;
bool _lookingInMirror; bool _lookingInMirror;
glm::vec3 _ballColor; glm::vec3 _ballColor;
std::vector<HandBall> _leapBalls; std::vector<HandBall> _leapFingerTipBalls;
std::vector<HandBall> _leapFingerRootBalls;
// private methods // private methods
void setLeapHands(const std::vector<glm::vec3>& handPositions, void setLeapHands(const std::vector<glm::vec3>& handPositions,
@ -92,8 +94,10 @@ private:
void renderRaveGloveStage(); void renderRaveGloveStage();
void setRaveGloveMode(int mode); void setRaveGloveMode(int mode);
void renderHandSpheres(); void renderLeapHandSpheres();
void renderFingerTrails(); void renderLeapHands();
void renderLeapHand(PalmData& hand);
void renderLeapFingerTrails();
void calculateGeometry(); void calculateGeometry();
}; };

4
libraries/avatars/CMakeLists.txt
View file

@ -8,13 +8,9 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cm
set(TARGET_NAME avatars) set(TARGET_NAME avatars)
find_package(Qt5Core)
include(${MACRO_DIR}/SetupHifiLibrary.cmake) include(${MACRO_DIR}/SetupHifiLibrary.cmake)
setup_hifi_library(${TARGET_NAME}) setup_hifi_library(${TARGET_NAME})
qt5_use_modules(${TARGET_NAME} Core)
include(${MACRO_DIR}/IncludeGLM.cmake) include(${MACRO_DIR}/IncludeGLM.cmake)
include_glm(${TARGET_NAME} ${ROOT_DIR}) include_glm(${TARGET_NAME} ${ROOT_DIR})

17
libraries/avatars/src/AvatarData.cpp Executable file → Normal file
View file

@ -130,6 +130,8 @@ int AvatarData::getBroadcastData(unsigned char* destinationBuffer) {
// leap hand data // leap hand data
std::vector<glm::vec3> fingerVectors; std::vector<glm::vec3> fingerVectors;
//printf("about to call _handData->encodeRemoteData(fingerVectors);\n");
_handData->encodeRemoteData(fingerVectors); _handData->encodeRemoteData(fingerVectors);
if (fingerVectors.size() > 255) if (fingerVectors.size() > 255)
@ -244,17 +246,32 @@ int AvatarData::parseData(unsigned char* sourceBuffer, int numBytes) {
// hand state, stored as a semi-nibble in the bitItems // hand state, stored as a semi-nibble in the bitItems
_handState = getSemiNibbleAt(bitItems,HAND_STATE_START_BIT); _handState = getSemiNibbleAt(bitItems,HAND_STATE_START_BIT);
//printf("about to call leap hand data code in AvatarData::parseData...\n");
// leap hand data // leap hand data
if (sourceBuffer - startPosition < numBytes) { if (sourceBuffer - startPosition < numBytes) {
//printf("got inside of 'if (sourceBuffer - startPosition < numBytes)'\n");
// check passed, bytes match // check passed, bytes match
unsigned int numFingerVectors = *sourceBuffer++; unsigned int numFingerVectors = *sourceBuffer++;
//printf("numFingerVectors = %d\n", numFingerVectors);
if (numFingerVectors > 0) { if (numFingerVectors > 0) {
//printf("ok, we got fingers in AvatarData::parseData\n");
std::vector<glm::vec3> fingerVectors(numFingerVectors); std::vector<glm::vec3> fingerVectors(numFingerVectors);
for (size_t i = 0; i < numFingerVectors; ++i) { for (size_t i = 0; i < numFingerVectors; ++i) {
sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].x), fingerVectorRadix); sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].x), fingerVectorRadix);
sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].y), fingerVectorRadix); sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].y), fingerVectorRadix);
sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].z), fingerVectorRadix); sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].z), fingerVectorRadix);
} }
//printf("about to call _handData->decodeRemoteData(fingerVectors);\n");
_handData->decodeRemoteData(fingerVectors); _handData->decodeRemoteData(fingerVectors);
} }
} }

9
libraries/avatars/src/HandData.cpp Executable file → Normal file
View file

@ -51,12 +51,18 @@ _owningHandData(owningHandData)
void HandData::encodeRemoteData(std::vector<glm::vec3>& fingerVectors) { void HandData::encodeRemoteData(std::vector<glm::vec3>& fingerVectors) {
fingerVectors.clear(); fingerVectors.clear();
for (size_t i = 0; i < getNumPalms(); ++i) { for (size_t i = 0; i < getNumPalms(); ++i) {
PalmData& palm = getPalms()[i]; PalmData& palm = getPalms()[i];
if (!palm.isActive()) {
continue;
}
fingerVectors.push_back(palm.getRawPosition()); fingerVectors.push_back(palm.getRawPosition());
fingerVectors.push_back(palm.getRawNormal()); fingerVectors.push_back(palm.getRawNormal());
for (size_t f = 0; f < palm.getNumFingers(); ++f) { for (size_t f = 0; f < palm.getNumFingers(); ++f) {
FingerData& finger = palm.getFingers()[f]; FingerData& finger = palm.getFingers()[f];
if (finger.isActive()) { if (finger.isActive()) {
fingerVectors.push_back(finger.getTipRawPosition()); fingerVectors.push_back(finger.getTipRawPosition());
fingerVectors.push_back(finger.getRootRawPosition()); fingerVectors.push_back(finger.getRootRawPosition());
@ -83,7 +89,8 @@ void HandData::decodeRemoteData(const std::vector<glm::vec3>& fingerVectors) {
palm.setRawPosition(fingerVectors[vectorIndex++]); palm.setRawPosition(fingerVectors[vectorIndex++]);
palm.setRawNormal(fingerVectors[vectorIndex++]); palm.setRawNormal(fingerVectors[vectorIndex++]);
for (size_t f = 0; f < NUM_FINGERS_PER_HAND; ++f) { for (size_t f = 0; f < NUM_FINGERS_PER_HAND; ++f) {
FingerData& finger = palm.getFingers()[i]; FingerData& finger = palm.getFingers()[f];
finger.setActive(true);
finger.setRawTipPosition(fingerVectors[vectorIndex++]); finger.setRawTipPosition(fingerVectors[vectorIndex++]);
finger.setRawRootPosition(fingerVectors[vectorIndex++]); finger.setRawRootPosition(fingerVectors[vectorIndex++]);
} }

4
libraries/shared/CMakeLists.txt
View file

@ -6,13 +6,9 @@ set(MACRO_DIR ${ROOT_DIR}/cmake/macros)
set(TARGET_NAME shared) set(TARGET_NAME shared)
project(${TARGET_NAME}) project(${TARGET_NAME})
find_package(Qt5Core REQUIRED)
include(${MACRO_DIR}/SetupHifiLibrary.cmake) include(${MACRO_DIR}/SetupHifiLibrary.cmake)
setup_hifi_library(${TARGET_NAME}) setup_hifi_library(${TARGET_NAME})
qt5_use_modules(${TARGET_NAME} Core)
set(EXTERNAL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external) set(EXTERNAL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external)
if (WIN32) if (WIN32)

4
libraries/voxels/CMakeLists.txt
View file

@ -8,9 +8,13 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cm
set(TARGET_NAME voxels) set(TARGET_NAME voxels)
find_package(Qt5Widgets REQUIRED)
include(${MACRO_DIR}/SetupHifiLibrary.cmake) include(${MACRO_DIR}/SetupHifiLibrary.cmake)
setup_hifi_library(${TARGET_NAME}) setup_hifi_library(${TARGET_NAME})
qt5_use_modules(${TARGET_NAME} Widgets)
include(${MACRO_DIR}/IncludeGLM.cmake) include(${MACRO_DIR}/IncludeGLM.cmake)
include_glm(${TARGET_NAME} ${ROOT_DIR}) include_glm(${TARGET_NAME} ${ROOT_DIR})

2
libraries/voxels/src/VoxelConstants.h
View file

@ -41,4 +41,6 @@ const glBufferIndex GLBUFFER_INDEX_UNKNOWN = ULONG_MAX;
const float SIXTY_FPS_IN_MILLISECONDS = 1000.0f / 60.0f; const float SIXTY_FPS_IN_MILLISECONDS = 1000.0f / 60.0f;
const float VIEW_CULLING_RATE_IN_MILLISECONDS = 1000.0f; // once a second is fine const float VIEW_CULLING_RATE_IN_MILLISECONDS = 1000.0f; // once a second is fine
const uint64_t CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS = 1000 * 5; // 1 packet every 50 milliseconds
#endif #endif

15
voxel-server/src/main.cpp
View file

@ -61,6 +61,7 @@ bool wantColorRandomizer = false;
bool debugVoxelSending = false; bool debugVoxelSending = false;
bool shouldShowAnimationDebug = false; bool shouldShowAnimationDebug = false;
bool displayVoxelStats = false; bool displayVoxelStats = false;
bool debugVoxelReceiving = false;
EnvironmentData environmentData[3]; EnvironmentData environmentData[3];
@ -426,6 +427,8 @@ void attachVoxelNodeDataToNode(Node* newNode) {
} }
} }
int receivedPacketCount = 0;
int main(int argc, const char * argv[]) { int main(int argc, const char * argv[]) {
pthread_mutex_init(&::treeLock, NULL); pthread_mutex_init(&::treeLock, NULL);
@ -455,6 +458,10 @@ int main(int argc, const char * argv[]) {
::debugVoxelSending = cmdOptionExists(argc, argv, DEBUG_VOXEL_SENDING); ::debugVoxelSending = cmdOptionExists(argc, argv, DEBUG_VOXEL_SENDING);
printf("debugVoxelSending=%s\n", debug::valueOf(::debugVoxelSending)); printf("debugVoxelSending=%s\n", debug::valueOf(::debugVoxelSending));
const char* DEBUG_VOXEL_RECEIVING = "--debugVoxelReceiving";
::debugVoxelReceiving = cmdOptionExists(argc, argv, DEBUG_VOXEL_RECEIVING);
printf("debugVoxelReceiving=%s\n", debug::valueOf(::debugVoxelReceiving));
const char* WANT_ANIMATION_DEBUG = "--shouldShowAnimationDebug"; const char* WANT_ANIMATION_DEBUG = "--shouldShowAnimationDebug";
::shouldShowAnimationDebug = cmdOptionExists(argc, argv, WANT_ANIMATION_DEBUG); ::shouldShowAnimationDebug = cmdOptionExists(argc, argv, WANT_ANIMATION_DEBUG);
printf("shouldShowAnimationDebug=%s\n", debug::valueOf(::shouldShowAnimationDebug)); printf("shouldShowAnimationDebug=%s\n", debug::valueOf(::shouldShowAnimationDebug));
@ -584,12 +591,20 @@ int main(int argc, const char * argv[]) {
destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
::shouldShowAnimationDebug); ::shouldShowAnimationDebug);
::receivedPacketCount++;
unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader))); unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader)));
if (::shouldShowAnimationDebug) { if (::shouldShowAnimationDebug) {
printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n", printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n",
destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
receivedBytes,itemNumber); receivedBytes,itemNumber);
} }
if (::debugVoxelReceiving) {
printf("got %s - %d command from client receivedBytes=%ld itemNumber=%d\n",
destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
::receivedPacketCount, receivedBytes,itemNumber);
}
int atByte = numBytesPacketHeader + sizeof(itemNumber); int atByte = numBytesPacketHeader + sizeof(itemNumber);
unsigned char* voxelData = (unsigned char*)&packetData[atByte]; unsigned char* voxelData = (unsigned char*)&packetData[atByte];
while (atByte < receivedBytes) { while (atByte < receivedBytes) {