diff --git a/interface/resources/shaders/face.vert b/interface/resources/shaders/face.vert index 358a8902fe..018a85f524 100644 --- a/interface/resources/shaders/face.vert +++ b/interface/resources/shaders/face.vert @@ -17,9 +17,6 @@ uniform vec2 texCoordRight; // the texture coordinate vector from bottom to the top uniform vec2 texCoordUp; -// the aspect ratio of the image -uniform float aspectRatio; - // the depth texture uniform sampler2D depthTexture; @@ -31,6 +28,5 @@ void main(void) { const float MIN_VISIBLE_DEPTH = 1.0 / 255.0; const float MAX_VISIBLE_DEPTH = 254.0 / 255.0; gl_FrontColor = vec4(1.0, 1.0, 1.0, step(MIN_VISIBLE_DEPTH, depth) * (1.0 - step(MAX_VISIBLE_DEPTH, depth))); - gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, - (gl_Vertex.y - 0.5) / aspectRatio, depth * 2.0 - 2.0, 1.0); + gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, gl_Vertex.y - 0.5, depth - 0.5, 1.0); } diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index b5ffb9d738..3d741fbfbb 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -196,8 +196,9 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _isTouchPressed(false), _yawFromTouch(0.0f), _pitchFromTouch(0.0f), - _groundPlaneImpact(0.0f), _mousePressed(false), + _isHoverVoxel(false), + _isHoverVoxelSounding(false), _mouseVoxelScale(1.0f / 1024.0f), _justEditedVoxel(false), _isLookingAtOtherAvatar(false), @@ -856,6 +857,11 @@ void Application::mouseMoveEvent(QMouseEvent* event) { } } +const bool MAKE_SOUND_ON_VOXEL_HOVER = false; +const bool MAKE_SOUND_ON_VOXEL_CLICK = true; +const float HOVER_VOXEL_FREQUENCY = 14080.f; +const float HOVER_VOXEL_DECAY = 0.999f; + void Application::mousePressEvent(QMouseEvent* event) { if (activeWindow() == _window) { if (event->button() == Qt::LeftButton) { @@ -866,9 +872,19 @@ void Application::mousePressEvent(QMouseEvent* event) { _mouseVoxelDragging = _mouseVoxel; _mousePressed = true; + if (!maybeEditVoxelUnderCursor()) { _pieMenu.mousePressEvent(_mouseX, _mouseY); } + + if (MAKE_SOUND_ON_VOXEL_CLICK && _isHoverVoxel && !_isHoverVoxelSounding) { + _hoverVoxelOriginalColor[0] = _hoverVoxel.red; + _hoverVoxelOriginalColor[1] = _hoverVoxel.green; + _hoverVoxelOriginalColor[2] = _hoverVoxel.blue; + _hoverVoxelOriginalColor[3] = 1; + _audio.startCollisionSound(1.0, HOVER_VOXEL_FREQUENCY * _hoverVoxel.s * TREE_SCALE, 0.0, HOVER_VOXEL_DECAY); + _isHoverVoxelSounding = true; + } } else if (event->button() == Qt::RightButton && checkedVoxelModeAction() != 0) { deleteVoxelUnderCursor(); @@ -925,8 +941,10 @@ void Application::touchEndEvent(QTouchEvent* event) { _isTouchPressed = false; } +const bool USE_MOUSEWHEEL = false; void Application::wheelEvent(QWheelEvent* event) { - if (activeWindow() == _window) { + // Wheel Events disabled for now because they are also activated by touch look pitch up/down. + if (USE_MOUSEWHEEL && (activeWindow() == _window)) { if (checkedVoxelModeAction() == 0) { event->ignore(); return; @@ -972,14 +990,15 @@ void Application::sendAvatarFaceVideoMessage(int frameCount, const QByteArray& d int headerSize = packetPosition - packet; - // break the data up into submessages of the maximum size + // break the data up into submessages of the maximum size (at least one, for zero-length packets) *offsetPosition = 0; - while (*offsetPosition < data.size()) { + do { int payloadSize = min(data.size() - (int)*offsetPosition, MAX_PACKET_SIZE - headerSize); memcpy(packetPosition, data.constData() + *offsetPosition, payloadSize); getInstance()->controlledBroadcastToNodes(packet, headerSize + payloadSize, &NODE_TYPE_AVATAR_MIXER, 1); *offsetPosition += payloadSize; - } + + } while (*offsetPosition < data.size()); } // Every second, check the frame rates and other stuff @@ -1502,16 +1521,18 @@ bool Application::sendVoxelsOperation(VoxelNode* node, void* extraData) { uint64_t now = usecTimestampNow(); // dynamically sleep until we need to fire off the next set of voxels - const uint64_t CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS = 1000 * 5; // 1 packet every 10 milliseconds uint64_t elapsed = now - args->lastSendTime; int usecToSleep = CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS - elapsed; if (usecToSleep > 0) { - qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", - args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed, usecToSleep); + //qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", + // args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed, usecToSleep); + + Application::getInstance()->timer(); + usleep(usecToSleep); } else { - qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", - args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed); + //qDebug("sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", + // args->packetsSent, (long long int)args->bytesSent, (long long int)elapsed); } args->lastSendTime = now; } @@ -1579,70 +1600,127 @@ void Application::importVoxelsToClipboard() { void Application::importVoxels() { QString desktopLocation = QStandardPaths::writableLocation(QStandardPaths::DesktopLocation); - QString fileNameString = QFileDialog::getOpenFileName(_glWidget, tr("Import Voxels"), desktopLocation, + + QStringList fileNameStringList = QFileDialog::getOpenFileNames(_glWidget, tr("Import Voxels"), desktopLocation, tr(IMPORT_FILE_TYPES)); - QByteArray fileNameAscii = fileNameString.toLocal8Bit(); - const char* fileName = fileNameAscii.data(); + + // remember the "selected" voxel point before we do any importing... + float originalX = _mouseVoxel.x; + float originalZ = _mouseVoxel.z; + + const int PNG_TYPE_NAME_LENGTH = 4; + const int SVO_TYPE_NAME_LENGTH = 4; + const int SCH_TYPE_NAME_LENGTH = 10; + + for (int i = 0; i < fileNameStringList.size(); i++) { + QString fileNameString = fileNameStringList.at(i); + QByteArray fileNameAscii = fileNameString.toLocal8Bit(); + const char* fileName = fileNameAscii.data(); - VoxelTree importVoxels; - if (fileNameString.endsWith(".png", Qt::CaseInsensitive)) { - QImage pngImage = QImage(fileName); - if (pngImage.height() != pngImage.width()) { - qDebug("ERROR: Bad PNG size: height != width.\n"); - return; + int fileTypeNameLength = 0; + VoxelTree importVoxels; + if (fileNameString.endsWith(".png", Qt::CaseInsensitive)) { + QImage pngImage = QImage(fileName); + fileTypeNameLength = PNG_TYPE_NAME_LENGTH; + if (pngImage.height() != pngImage.width()) { + qDebug("ERROR: Bad PNG size: height != width.\n"); + return; + } + + const uint32_t* pixels; + if (pngImage.format() == QImage::Format_ARGB32) { + pixels = reinterpret_cast(pngImage.constBits()); + } else { + QImage tmp = pngImage.convertToFormat(QImage::Format_ARGB32); + pixels = reinterpret_cast(tmp.constBits()); + } + + importVoxels.readFromSquareARGB32Pixels(pixels, pngImage.height()); + } else if (fileNameString.endsWith(".svo", Qt::CaseInsensitive)) { + importVoxels.readFromSVOFile(fileName); + fileTypeNameLength = SVO_TYPE_NAME_LENGTH; + } else if (fileNameString.endsWith(".schematic", Qt::CaseInsensitive)) { + importVoxels.readFromSchematicFile(fileName); + fileTypeNameLength = SCH_TYPE_NAME_LENGTH; } - const uint32_t* pixels; - if (pngImage.format() == QImage::Format_ARGB32) { - pixels = reinterpret_cast(pngImage.constBits()); + int indexOfFirstPeriod = fileNameString.indexOf('.'); + + QString fileCoord = fileNameString.mid(indexOfFirstPeriod + 1, + fileNameString.length() - indexOfFirstPeriod - fileTypeNameLength - 1); + + indexOfFirstPeriod = fileCoord.indexOf('.'); + QString columnNumString = fileCoord.right(fileCoord.length() - indexOfFirstPeriod - 1); + QString rowNumString = fileCoord.left(indexOfFirstPeriod); + + int columnNum = columnNumString.toFloat(); + int rowNum = rowNumString.toFloat(); + + qDebug("columnNum: %d\t rowNum: %d\n", columnNum, rowNum); + + _mouseVoxel.x = originalX + (columnNum - 1) * _mouseVoxel.s; + _mouseVoxel.z = originalZ + (rowNum - 1) * _mouseVoxel.s; + + VoxelNode* selectedNode = _voxels.getVoxelAt(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); + + // Recurse the Import Voxels tree, where everything is root relative, and send all the colored voxels to + // the server as an set voxel message, this will also rebase the voxels to the new location + unsigned char* calculatedOctCode = NULL; + SendVoxelsOperationArgs args; + args.lastSendTime = usecTimestampNow(); + args.packetsSent = 0; + args.bytesSent = 0; + + int numBytesPacketHeader = populateTypeAndVersion(args.messageBuffer, PACKET_TYPE_SET_VOXEL_DESTRUCTIVE); + + unsigned short int* sequenceAt = (unsigned short int*)&args.messageBuffer[numBytesPacketHeader]; + *sequenceAt = 0; + args.bufferInUse = numBytesPacketHeader + sizeof(unsigned short int); // set to command + sequence + + // we only need the selected voxel to get the newBaseOctCode, which we can actually calculate from the + // voxel size/position details. + if (selectedNode) { + args.newBaseOctCode = selectedNode->getOctalCode(); } else { - QImage tmp = pngImage.convertToFormat(QImage::Format_ARGB32); - pixels = reinterpret_cast(tmp.constBits()); + args.newBaseOctCode = calculatedOctCode = pointToVoxel(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); + } + + qDebug("column:%d, row:%d, voxel:%f,%f,%f,%f\n", columnNum, rowNum, _mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s ); + + // send the insert/paste of these voxels + importVoxels.recurseTreeWithOperation(sendVoxelsOperation, &args); + + // If we have voxels left in the packet, then send the packet + if (args.bufferInUse > (numBytesPacketHeader + sizeof(unsigned short int))) { + controlledBroadcastToNodes(args.messageBuffer, args.bufferInUse, & NODE_TYPE_VOXEL_SERVER, 1); + + + args.packetsSent++; + args.bytesSent += args.bufferInUse; + + uint64_t now = usecTimestampNow(); + // dynamically sleep until we need to fire off the next set of voxels + uint64_t elapsed = now - args.lastSendTime; + int usecToSleep = CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS - elapsed; + + if (usecToSleep > 0) { + //qDebug("after sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, sleeping for %d usecs!\n", + // args.packetsSent, (long long int)args.bytesSent, (long long int)elapsed, usecToSleep); + usleep(usecToSleep); + } else { + //qDebug("after sendVoxelsOperation: packet: %d bytes:%lld elapsed %lld usecs, no need to sleep!\n", + // args.packetsSent, (long long int)args.bytesSent, (long long int)elapsed); + } + args.lastSendTime = now; } - importVoxels.readFromSquareARGB32Pixels(pixels, pngImage.height()); - } else if (fileNameString.endsWith(".svo", Qt::CaseInsensitive)) { - importVoxels.readFromSVOFile(fileName); - } else if (fileNameString.endsWith(".schematic", Qt::CaseInsensitive)) { - importVoxels.readFromSchematicFile(fileName); + if (calculatedOctCode) { + delete[] calculatedOctCode; + } + } - VoxelNode* selectedNode = _voxels.getVoxelAt(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); - - // Recurse the Import Voxels tree, where everything is root relative, and send all the colored voxels to - // the server as an set voxel message, this will also rebase the voxels to the new location - unsigned char* calculatedOctCode = NULL; - SendVoxelsOperationArgs args; - args.lastSendTime = usecTimestampNow(); - args.packetsSent = 0; - args.bytesSent = 0; - - int numBytesPacketHeader = populateTypeAndVersion(args.messageBuffer, PACKET_TYPE_SET_VOXEL_DESTRUCTIVE); - - unsigned short int* sequenceAt = (unsigned short int*)&args.messageBuffer[numBytesPacketHeader]; - *sequenceAt = 0; - args.bufferInUse = numBytesPacketHeader + sizeof(unsigned short int); // set to command + sequence - - // we only need the selected voxel to get the newBaseOctCode, which we can actually calculate from the - // voxel size/position details. - if (selectedNode) { - args.newBaseOctCode = selectedNode->getOctalCode(); - } else { - args.newBaseOctCode = calculatedOctCode = pointToVoxel(_mouseVoxel.x, _mouseVoxel.y, _mouseVoxel.z, _mouseVoxel.s); - } - - importVoxels.recurseTreeWithOperation(sendVoxelsOperation, &args); - - // If we have voxels left in the packet, then send the packet - if (args.bufferInUse > (numBytesPacketHeader + sizeof(unsigned short int))) { - controlledBroadcastToNodes(args.messageBuffer, args.bufferInUse, & NODE_TYPE_VOXEL_SERVER, 1); - } - - if (calculatedOctCode) { - delete[] calculatedOctCode; - } - // restore the main window's active state _window->activateWindow(); } @@ -1735,6 +1813,8 @@ void Application::initMenu() { _testPing->setChecked(true); (_fullScreenMode = optionsMenu->addAction("Fullscreen", this, SLOT(setFullscreen(bool)), Qt::Key_F))->setCheckable(true); optionsMenu->addAction("Webcam", &_webcam, SLOT(setEnabled(bool)))->setCheckable(true); + optionsMenu->addAction("Toggle Skeleton Tracking", &_webcam, SLOT(setSkeletonTrackingOn(bool)))->setCheckable(true); + optionsMenu->addAction("Cycle Webcam Send Mode", _webcam.getGrabber(), SLOT(cycleVideoSendMode())); optionsMenu->addAction("Go Home", this, SLOT(goHome())); QMenu* renderMenu = menuBar->addMenu("Render"); @@ -1762,7 +1842,6 @@ void Application::initMenu() { (_renderLookatIndicatorOn = renderMenu->addAction("Lookat Indicator"))->setCheckable(true); _renderLookatIndicatorOn->setChecked(true); (_renderParticleSystemOn = renderMenu->addAction("Particle System"))->setCheckable(true); - _renderParticleSystemOn->setChecked(true); (_manualFirstPerson = renderMenu->addAction( "First Person", this, SLOT(setRenderFirstPerson(bool)), Qt::Key_P))->setCheckable(true); (_manualThirdPerson = renderMenu->addAction( @@ -2035,6 +2114,7 @@ void Application::renderLookatIndicator(glm::vec3 pointOfInterest, Camera& which } void Application::update(float deltaTime) { + // Use Transmitter Hand to move hand if connected, else use mouse if (_myTransmitter.isConnected()) { const float HAND_FORCE_SCALING = 0.01f; @@ -2067,11 +2147,46 @@ void Application::update(float deltaTime) { // If the mouse is over another avatar's head... glm::vec3 myLookAtFromMouse(eyePosition); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); + } else if (_isHoverVoxel) { + // Look at the hovered voxel + glm::vec3 lookAtSpot = getMouseVoxelWorldCoordinates(_hoverVoxel); + _myAvatar.getHead().setLookAtPosition(lookAtSpot); } else { + // Just look in direction of the mouse ray glm::vec3 myLookAtFromMouse(mouseRayOrigin + mouseRayDirection); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); } - + + // Find the voxel we are hovering over, and respond if clicked + float distance; + BoxFace face; + + // If we have clicked on a voxel, update it's color + if (_isHoverVoxelSounding) { + VoxelNode* hoveredNode = _voxels.getVoxelAt(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s); + float bright = _audio.getCollisionSoundMagnitude(); + nodeColor clickColor = { 255 * bright + _hoverVoxelOriginalColor[0] * (1.f - bright), + _hoverVoxelOriginalColor[1] * (1.f - bright), + _hoverVoxelOriginalColor[2] * (1.f - bright), 1 }; + hoveredNode->setColor(clickColor); + if (bright < 0.01f) { + hoveredNode->setColor(_hoverVoxelOriginalColor); + _isHoverVoxelSounding = false; + } + } else { + // Check for a new hover voxel + glm::vec4 oldVoxel(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s); + _isHoverVoxel = _voxels.findRayIntersection(mouseRayOrigin, mouseRayDirection, _hoverVoxel, distance, face); + if (MAKE_SOUND_ON_VOXEL_HOVER && _isHoverVoxel && glm::vec4(_hoverVoxel.x, _hoverVoxel.y, _hoverVoxel.z, _hoverVoxel.s) != oldVoxel) { + _hoverVoxelOriginalColor[0] = _hoverVoxel.red; + _hoverVoxelOriginalColor[1] = _hoverVoxel.green; + _hoverVoxelOriginalColor[2] = _hoverVoxel.blue; + _hoverVoxelOriginalColor[3] = 1; + _audio.startCollisionSound(1.0, HOVER_VOXEL_FREQUENCY * _hoverVoxel.s * TREE_SCALE, 0.0, HOVER_VOXEL_DECAY); + _isHoverVoxelSounding = true; + } + } + // If we are dragging on a voxel, add thrust according to the amount the mouse is dragging const float VOXEL_GRAB_THRUST = 0.0f; if (_mousePressed && (_mouseVoxel.s != 0)) { @@ -2097,8 +2212,6 @@ void Application::update(float deltaTime) { (fabs(_myAvatar.getVelocity().x) + fabs(_myAvatar.getVelocity().y) + fabs(_myAvatar.getVelocity().z)) / 3 < MAX_AVATAR_EDIT_VELOCITY) { - float distance; - BoxFace face; if (_voxels.findRayIntersection(mouseRayOrigin, mouseRayDirection, _mouseVoxel, distance, face)) { if (distance < MAX_VOXEL_EDIT_DISTANCE) { // find the nearest voxel with the desired scale @@ -2285,20 +2398,18 @@ void Application::update(float deltaTime) { } void Application::updateAvatar(float deltaTime) { - - // When head is rotated via touch/mouse look, slowly turn body to follow - const float BODY_FOLLOW_HEAD_RATE = 0.5f; - // update body yaw by body yaw delta + + // rotate body yaw for yaw received from multitouch _myAvatar.setOrientation(_myAvatar.getOrientation() - * glm::quat(glm::vec3(0, _yawFromTouch * deltaTime * BODY_FOLLOW_HEAD_RATE, 0) * deltaTime)); - _yawFromTouch -= _yawFromTouch * deltaTime * BODY_FOLLOW_HEAD_RATE; + * glm::quat(glm::vec3(0, _yawFromTouch * deltaTime, 0))); + _yawFromTouch = 0.f; // Update my avatar's state from gyros and/or webcam _myAvatar.updateFromGyrosAndOrWebcam(_gyroLook->isChecked(), glm::vec3(_headCameraPitchYawScale, _headCameraPitchYawScale, _headCameraPitchYawScale), - _yawFromTouch, + 0.f, _pitchFromTouch); if (_serialHeadSensor.isActive()) { @@ -2757,8 +2868,12 @@ void Application::displayOverlay() { glDisable(GL_DEPTH_TEST); glDisable(GL_LIGHTING); - // Display a single screen-size quad to - renderCollisionOverlay(_glWidget->width(), _glWidget->height(), _audio.getCollisionSoundMagnitude()); + // Display a single screen-size quad to create an alpha blended 'collision' flash + float collisionSoundMagnitude = _audio.getCollisionSoundMagnitude(); + const float VISIBLE_COLLISION_SOUND_MAGNITUDE = 0.5f; + if (collisionSoundMagnitude > VISIBLE_COLLISION_SOUND_MAGNITUDE) { + renderCollisionOverlay(_glWidget->width(), _glWidget->height(), _audio.getCollisionSoundMagnitude()); + } #ifndef _WIN32 _audio.render(_glWidget->width(), _glWidget->height()); @@ -3481,6 +3596,7 @@ void* Application::networkReceive(void* args) { } if (NodeList::getInstance()->getNodeSocket()->receive(&senderAddress, app->_incomingPacket, &bytesReceived)) { + app->_packetCount++; app->_bytesCount += bytesReceived; @@ -3525,13 +3641,11 @@ void* Application::networkReceive(void* args) { Node* voxelServer = NodeList::getInstance()->soloNodeOfType(NODE_TYPE_VOXEL_SERVER); if (voxelServer && socketMatch(voxelServer->getActiveSocket(), &senderAddress)) { voxelServer->lock(); - if (messageData[0] == PACKET_TYPE_ENVIRONMENT_DATA) { app->_environment.parseData(&senderAddress, messageData, messageLength); } else { app->_voxels.parseData(messageData, messageLength); } - voxelServer->unlock(); } } diff --git a/interface/src/Application.h b/interface/src/Application.h index fbecebd0b5..be8f0a7e3b 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -110,8 +110,6 @@ public slots: void sendAvatarFaceVideoMessage(int frameCount, const QByteArray& data); - void setGroundPlaneImpact(float groundPlaneImpact) { _groundPlaneImpact = groundPlaneImpact; } - private slots: @@ -372,14 +370,16 @@ private: float _yawFromTouch; float _pitchFromTouch; - float _groundPlaneImpact; - VoxelDetail _mouseVoxelDragging; glm::vec3 _voxelThrust; bool _mousePressed; // true if mouse has been pressed (clear when finished) + VoxelDetail _hoverVoxel; // Stuff about the voxel I am hovering or clicking + bool _isHoverVoxel; + bool _isHoverVoxelSounding; + nodeColor _hoverVoxelOriginalColor; - VoxelDetail _mouseVoxel; // details of the voxel under the mouse cursor + VoxelDetail _mouseVoxel; // details of the voxel to be edited float _mouseVoxelScale; // the scale for adding/removing voxels glm::vec3 _lastMouseVoxelPos; // the position of the last mouse voxel edit bool _justEditedVoxel; // set when we've just added/deleted/colored a voxel diff --git a/interface/src/ParticleSystem.cpp b/interface/src/ParticleSystem.cpp index 5e2b92bb64..fcea8ca270 100644 --- a/interface/src/ParticleSystem.cpp +++ b/interface/src/ParticleSystem.cpp @@ -27,6 +27,7 @@ ParticleSystem::ParticleSystem() { for (unsigned int emitterIndex = 0; emitterIndex < MAX_EMITTERS; emitterIndex++) { Emitter * e = &_emitter[emitterIndex]; + e->active = false; e->position = glm::vec3(0.0f, 0.0f, 0.0f); e->previousPosition = glm::vec3(0.0f, 0.0f, 0.0f); e->direction = glm::vec3(0.0f, 1.0f, 0.0f); @@ -72,25 +73,16 @@ void ParticleSystem::simulate(float deltaTime) { _timer += deltaTime; - // emit particles - for (int e = 0; e < _numEmitters; e++) { + // update emitters + for (int emitterIndex = 0; emitterIndex < _numEmitters; emitterIndex++) { + assert(emitterIndex <= MAX_EMITTERS); - assert(e >= 0); - assert(e <= MAX_EMITTERS); - assert(_emitter[e].rate >= 0); - - _emitter[e].emitReserve += _emitter[e].rate * deltaTime; - _emitter[e].numParticlesEmittedThisTime = (int)_emitter[e].emitReserve; - _emitter[e].emitReserve -= _emitter[e].numParticlesEmittedThisTime; - - for (int p = 0; p < _emitter[e].numParticlesEmittedThisTime; p++) { - float timeFraction = (float)p / (float)_emitter[e].numParticlesEmittedThisTime; - createParticle(e, timeFraction); + if (_emitter[emitterIndex].active) { + updateEmitter(emitterIndex, deltaTime); } } - - // update particles + // update particles for (int p = 0; p < MAX_PARTICLES; p++) { if (_particle[p].alive) { if (_particle[p].age > _emitter[_particle[p].emitterIndex].particleLifespan) { @@ -102,6 +94,20 @@ void ParticleSystem::simulate(float deltaTime) { } } + +void ParticleSystem::updateEmitter(int emitterIndex, float deltaTime) { + + _emitter[emitterIndex].emitReserve += _emitter[emitterIndex].rate * deltaTime; + _emitter[emitterIndex].numParticlesEmittedThisTime = (int)_emitter[emitterIndex].emitReserve; + _emitter[emitterIndex].emitReserve -= _emitter[emitterIndex].numParticlesEmittedThisTime; + + for (int p = 0; p < _emitter[emitterIndex].numParticlesEmittedThisTime; p++) { + float timeFraction = (float)p / (float)_emitter[emitterIndex].numParticlesEmittedThisTime; + createParticle(emitterIndex, timeFraction); + } +} + + void ParticleSystem::createParticle(int e, float timeFraction) { for (unsigned int p = 0; p < MAX_PARTICLES; p++) { @@ -212,7 +218,6 @@ void ParticleSystem::setParticleAttributes(int emitterIndex, ParticleLifeStage l } - void ParticleSystem::updateParticle(int p, float deltaTime) { Emitter myEmitter = _emitter[_particle[p].emitterIndex]; @@ -363,14 +368,16 @@ void ParticleSystem::killAllParticles() { void ParticleSystem::render() { // render the emitters - for (int e = 0; e < _numEmitters; e++) { + for (int e = 0; e < MAX_EMITTERS; e++) { - if (_emitter[e].showingBaseParticle) { - glColor4f(_particle[0].color.r, _particle[0].color.g, _particle[0].color.b, _particle[0].color.a); - glPushMatrix(); - glTranslatef(_emitter[e].position.x, _emitter[e].position.y, _emitter[e].position.z); - glutSolidSphere(_particle[0].radius, _emitter[e].particleResolution, _emitter[e].particleResolution); - glPopMatrix(); + if (_emitter[e].active) { + if (_emitter[e].showingBaseParticle) { + glColor4f(_particle[0].color.r, _particle[0].color.g, _particle[0].color.b, _particle[0].color.a); + glPushMatrix(); + glTranslatef(_emitter[e].position.x, _emitter[e].position.y, _emitter[e].position.z); + glutSolidSphere(_particle[0].radius, _emitter[e].particleResolution, _emitter[e].particleResolution); + glPopMatrix(); + } } if (_emitter[e].visible) { diff --git a/interface/src/ParticleSystem.h b/interface/src/ParticleSystem.h index d79f621f69..a0131883a1 100644 --- a/interface/src/ParticleSystem.h +++ b/interface/src/ParticleSystem.h @@ -10,10 +10,10 @@ #include -const int MAX_PARTICLES = 5000; const int NULL_EMITTER = -1; const int NULL_PARTICLE = -1; const int MAX_EMITTERS = 100; +const int MAX_PARTICLES = 5000; enum ParticleRenderStyle { @@ -78,6 +78,7 @@ public: void setParticleAttributes (int emitterIndex, ParticleAttributes attributes); // set attributes for whole life of particles void setParticleAttributes (int emitterIndex, ParticleLifeStage lifeStage, ParticleAttributes attributes); // set attributes for this life stage void setEmitterPosition (int emitterIndex, glm::vec3 position ); + void setEmitterActive (int emitterIndex, bool active ) {_emitter[emitterIndex].active = active; } void setEmitterParticleResolution (int emitterIndex, int resolution ) {_emitter[emitterIndex].particleResolution = resolution; } void setEmitterDirection (int emitterIndex, glm::vec3 direction ) {_emitter[emitterIndex].direction = direction; } void setShowingEmitter (int emitterIndex, bool showing ) {_emitter[emitterIndex].visible = showing; } @@ -101,6 +102,7 @@ private: }; struct Emitter { + bool active; // if false, the emitter is disabled - allows for easy switching on and off glm::vec3 position; // the position of the emitter in world coordinates glm::vec3 previousPosition; // the position of the emitter in the previous time step glm::vec3 direction; // a normalized vector used as an axis for particle emission and other effects @@ -124,6 +126,7 @@ private: float _timer; // private methods + void updateEmitter(int emitterIndex, float deltaTime); void updateParticle(int index, float deltaTime); void createParticle(int e, float timeFraction); void killParticle(int p); diff --git a/interface/src/Physics.cpp b/interface/src/Physics.cpp index 31e64ccad3..101087b1ef 100644 --- a/interface/src/Physics.cpp +++ b/interface/src/Physics.cpp @@ -38,3 +38,7 @@ void applyDamping(float deltaTime, glm::vec3& velocity, float linearStrength, fl } } +void applyDampedSpring(float deltaTime, glm::vec3& velocity, glm::vec3& position, glm::vec3& targetPosition, float k, float damping) { + +} + diff --git a/interface/src/Util.cpp b/interface/src/Util.cpp index d2f9e56462..5d77c124f4 100644 --- a/interface/src/Util.cpp +++ b/interface/src/Util.cpp @@ -339,6 +339,7 @@ void renderCollisionOverlay(int width, int height, float magnitude) { } void renderGroundPlaneGrid(float size, float impact) { + float IMPACT_SOUND_MAGNITUDE_FOR_RECOLOR = 1.f; glLineWidth(2.0); glm::vec4 impactColor(1, 0, 0, 1); glm::vec3 lineColor(0.4, 0.5, 0.3); @@ -355,7 +356,12 @@ void renderGroundPlaneGrid(float size, float impact) { } // Draw the floor, colored for recent impact - glm::vec4 floorColor = impact * impactColor + (1.f - impact) * surfaceColor; + glm::vec4 floorColor; + if (impact > IMPACT_SOUND_MAGNITUDE_FOR_RECOLOR) { + floorColor = impact * impactColor + (1.f - impact) * surfaceColor; + } else { + floorColor = surfaceColor; + } glColor4fv(&floorColor.x); glBegin(GL_QUADS); glVertex3f(0, 0, 0); diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index d4fa015ba0..8fdf5f8687 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -19,6 +19,7 @@ #include "Application.h" #include "Webcam.h" +#include "avatar/Face.h" using namespace cv; using namespace std; @@ -32,7 +33,7 @@ int jointVectorMetaType = qRegisterMetaType("JointVector"); int matMetaType = qRegisterMetaType("cv::Mat"); int rotatedRectMetaType = qRegisterMetaType("cv::RotatedRect"); -Webcam::Webcam() : _enabled(false), _active(false), _colorTextureID(0), _depthTextureID(0) { +Webcam::Webcam() : _enabled(false), _active(false), _colorTextureID(0), _depthTextureID(0), _skeletonTrackingOn(false) { // the grabber simply runs as fast as possible _grabber = new FrameGrabber(); _grabber->moveToThread(&_grabberThread); @@ -154,8 +155,8 @@ Webcam::~Webcam() { const float METERS_PER_MM = 1.0f / 1000.0f; -void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float meanFaceDepth, - const RotatedRect& faceRect, const JointVector& joints) { +void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float midFaceDepth, + float aspectRatio, const RotatedRect& faceRect, bool sending, const JointVector& joints) { IplImage colorImage = color; glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); if (_colorTextureID == 0) { @@ -192,9 +193,11 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); glBindTexture(GL_TEXTURE_2D, 0); - // store our face rect and joints, update our frame count for fps computation + // store our various data, update our frame count for fps computation + _aspectRatio = aspectRatio; _faceRect = faceRect; - _joints = joints; + _sending = sending; + _joints = _skeletonTrackingOn ? joints : JointVector(); _frameCount++; const int MAX_FPS = 60; @@ -242,18 +245,18 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean if (_initialFaceRect.size.area() == 0) { _initialFaceRect = _faceRect; _estimatedPosition = glm::vec3(); - _initialFaceDepth = meanFaceDepth; + _initialFaceDepth = midFaceDepth; } else { float proportion, z; - if (meanFaceDepth == UNINITIALIZED_FACE_DEPTH) { + if (midFaceDepth == UNINITIALIZED_FACE_DEPTH) { proportion = sqrtf(_initialFaceRect.size.area() / (float)_faceRect.size.area()); const float INITIAL_DISTANCE_TO_CAMERA = 0.333f; z = INITIAL_DISTANCE_TO_CAMERA * proportion - INITIAL_DISTANCE_TO_CAMERA; } else { - z = (meanFaceDepth - _initialFaceDepth) * METERS_PER_MM; - proportion = meanFaceDepth / _initialFaceDepth; + z = (midFaceDepth - _initialFaceDepth) * METERS_PER_MM; + proportion = midFaceDepth / _initialFaceDepth; } const float POSITION_SCALE = 0.5f; _estimatedPosition = glm::vec3( @@ -270,8 +273,8 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, float mean QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame())); } -FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), - _smoothedMeanFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) { +FrameGrabber::FrameGrabber() : _initialized(false), _videoSendMode(FULL_FRAME_VIDEO), _capture(0), _searchWindow(0, 0, 0, 0), + _smoothedMidFaceDepth(UNINITIALIZED_FACE_DEPTH), _colorCodec(), _depthCodec(), _frameCount(0) { } FrameGrabber::~FrameGrabber() { @@ -364,6 +367,13 @@ static void XN_CALLBACK_TYPE calibrationCompleted(SkeletonCapability& capability } #endif +void FrameGrabber::cycleVideoSendMode() { + _videoSendMode = (VideoSendMode)((_videoSendMode + 1) % VIDEO_SEND_MODE_COUNT); + _searchWindow = cv::Rect(0, 0, 0, 0); + + destroyCodecs(); +} + void FrameGrabber::reset() { _searchWindow = cv::Rect(0, 0, 0, 0); @@ -379,14 +389,7 @@ void FrameGrabber::shutdown() { cvReleaseCapture(&_capture); _capture = 0; } - if (_colorCodec.name != 0) { - vpx_codec_destroy(&_colorCodec); - _colorCodec.name = 0; - } - if (_depthCodec.name != 0) { - vpx_codec_destroy(&_depthCodec); - _depthCodec.name = 0; - } + destroyCodecs(); _initialized = false; thread()->quit(); @@ -462,218 +465,208 @@ void FrameGrabber::grabFrame() { color = image; } - // if we don't have a search window (yet), try using the face cascade - int channels = 0; - float ranges[] = { 0, 180 }; - const float* range = ranges; - if (_searchWindow.area() == 0) { - vector faces; - _faceCascade.detectMultiScale(color, faces, 1.1, 6); - if (!faces.empty()) { - _searchWindow = faces.front(); - updateHSVFrame(color, format); - - Mat faceHsv(_hsvFrame, _searchWindow); - Mat faceMask(_mask, _searchWindow); - int sizes = 30; - calcHist(&faceHsv, 1, &channels, faceMask, _histogram, 1, &sizes, &range); - double min, max; - minMaxLoc(_histogram, &min, &max); - _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max); - } - } - RotatedRect faceRect; - if (_searchWindow.area() > 0) { - updateHSVFrame(color, format); - - calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range); - bitwise_and(_backProject, _mask, _backProject); - - faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1)); - Rect faceBounds = faceRect.boundingRect(); - Rect imageBounds(0, 0, color.cols, color.rows); - _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); - } - - const int ENCODED_FACE_WIDTH = 128; - const int ENCODED_FACE_HEIGHT = 128; - if (_colorCodec.name == 0) { - // initialize encoder context(s) - vpx_codec_enc_cfg_t codecConfig; - vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0); - codecConfig.rc_target_bitrate = ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * - codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h; - codecConfig.g_w = ENCODED_FACE_WIDTH; - codecConfig.g_h = ENCODED_FACE_HEIGHT; - vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0); - - if (!depth.empty()) { - int DEPTH_BITRATE_MULTIPLIER = 2; - codecConfig.rc_target_bitrate *= 2; - vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0); - } - } - - // correct for 180 degree rotations - if (faceRect.angle < -90.0f) { - faceRect.angle += 180.0f; - - } else if (faceRect.angle > 90.0f) { - faceRect.angle -= 180.0f; - } - - // compute the smoothed face rect - if (_smoothedFaceRect.size.area() == 0) { - _smoothedFaceRect = faceRect; + int encodedWidth; + int encodedHeight; + int depthBitrateMultiplier = 1; + Mat faceTransform; + float aspectRatio; + if (_videoSendMode == FULL_FRAME_VIDEO) { + // no need to find the face if we're sending full frame video + _smoothedFaceRect = RotatedRect(Point2f(color.cols / 2.0f, color.rows / 2.0f), Size2f(color.cols, color.rows), 0.0f); + encodedWidth = color.cols; + encodedHeight = color.rows; + aspectRatio = FULL_FRAME_ASPECT; } else { - const float FACE_RECT_SMOOTHING = 0.9f; - _smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING); - _smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING); - _smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING); - _smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING); - _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); - } - - // resize/rotate face into encoding rectangle - _faceColor.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_8UC3); - Point2f sourcePoints[4]; - _smoothedFaceRect.points(sourcePoints); - Point2f destPoints[] = { Point2f(0, ENCODED_FACE_HEIGHT), Point2f(0, 0), Point2f(ENCODED_FACE_WIDTH, 0) }; - Mat transform = getAffineTransform(sourcePoints, destPoints); - warpAffine(color, _faceColor, transform, _faceColor.size()); - - // convert from RGB to YV12 - const int ENCODED_BITS_PER_Y = 8; - const int ENCODED_BITS_PER_VU = 2; - const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU; - const int BITS_PER_BYTE = 8; - _encodedFace.resize(ENCODED_FACE_WIDTH * ENCODED_FACE_HEIGHT * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE); - vpx_image_t vpxImage; - vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, 1, (unsigned char*)_encodedFace.data()); - uchar* yline = vpxImage.planes[0]; - uchar* vline = vpxImage.planes[1]; - uchar* uline = vpxImage.planes[2]; - const int Y_RED_WEIGHT = (int)(0.299 * 256); - const int Y_GREEN_WEIGHT = (int)(0.587 * 256); - const int Y_BLUE_WEIGHT = (int)(0.114 * 256); - const int V_RED_WEIGHT = (int)(0.713 * 256); - const int U_BLUE_WEIGHT = (int)(0.564 * 256); - int redIndex = 0; - int greenIndex = 1; - int blueIndex = 2; - if (format == GL_BGR) { - redIndex = 2; - blueIndex = 0; - } - for (int i = 0; i < ENCODED_FACE_HEIGHT; i += 2) { - uchar* ydest = yline; - uchar* vdest = vline; - uchar* udest = uline; - for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { - uchar* tl = _faceColor.ptr(i, j); - uchar* tr = _faceColor.ptr(i, j + 1); - uchar* bl = _faceColor.ptr(i + 1, j); - uchar* br = _faceColor.ptr(i + 1, j + 1); + // if we don't have a search window (yet), try using the face cascade + int channels = 0; + float ranges[] = { 0, 180 }; + const float* range = ranges; + if (_searchWindow.area() == 0) { + vector faces; + _faceCascade.detectMultiScale(color, faces, 1.1, 6); + if (!faces.empty()) { + _searchWindow = faces.front(); + updateHSVFrame(color, format); - ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] * - Y_GREEN_WEIGHT + bl[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] * - Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8; - ydest += 2; - - int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex]; - int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex]; - int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex]; - int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8; - - *vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128; - *udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128; - } - yline += vpxImage.stride[0] * 2; - vline += vpxImage.stride[1]; - uline += vpxImage.stride[2]; - } - - // encode the frame - vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); - - // start the payload off with the aspect ratio - QByteArray payload(sizeof(float), 0); - *(float*)payload.data() = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; - - // extract the encoded frame - vpx_codec_iter_t iterator = 0; - const vpx_codec_cx_pkt_t* packet; - while ((packet = vpx_codec_get_cx_data(&_colorCodec, &iterator)) != 0) { - if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { - // prepend the length, which will indicate whether there's a depth frame too - payload.append((const char*)&packet->data.frame.sz, sizeof(packet->data.frame.sz)); - payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); - } - } - - if (!depth.empty()) { - // warp the face depth without interpolation (because it will contain invalid zero values) - _faceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_16UC1); - warpAffine(depth, _faceDepth, transform, _faceDepth.size(), INTER_NEAREST); - - // find the mean of the valid values - qint64 depthTotal = 0; - qint64 depthSamples = 0; - ushort* src = _faceDepth.ptr(); - const ushort ELEVEN_BIT_MINIMUM = 0; - const ushort ELEVEN_BIT_MAXIMUM = 2047; - for (int i = 0; i < ENCODED_FACE_HEIGHT; i++) { - for (int j = 0; j < ENCODED_FACE_WIDTH; j++) { - ushort depth = *src++; - if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { - depthTotal += depth; - depthSamples++; - } + Mat faceHsv(_hsvFrame, _searchWindow); + Mat faceMask(_mask, _searchWindow); + int sizes = 30; + calcHist(&faceHsv, 1, &channels, faceMask, _histogram, 1, &sizes, &range); + double min, max; + minMaxLoc(_histogram, &min, &max); + _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max); } } - float mean = (depthSamples == 0) ? UNINITIALIZED_FACE_DEPTH : depthTotal / (float)depthSamples; + RotatedRect faceRect; + if (_searchWindow.area() > 0) { + updateHSVFrame(color, format); + + calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range); + bitwise_and(_backProject, _mask, _backProject); + + faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1)); + Rect faceBounds = faceRect.boundingRect(); + Rect imageBounds(0, 0, color.cols, color.rows); + _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); + } + const int ENCODED_FACE_WIDTH = 128; + const int ENCODED_FACE_HEIGHT = 128; + encodedWidth = ENCODED_FACE_WIDTH; + encodedHeight = ENCODED_FACE_HEIGHT; + depthBitrateMultiplier = 2; - // smooth the mean over time - const float DEPTH_OFFSET_SMOOTHING = 0.95f; - _smoothedMeanFaceDepth = (_smoothedMeanFaceDepth == UNINITIALIZED_FACE_DEPTH) ? mean : - glm::mix(mean, _smoothedMeanFaceDepth, DEPTH_OFFSET_SMOOTHING); + // correct for 180 degree rotations + if (faceRect.angle < -90.0f) { + faceRect.angle += 180.0f; + + } else if (faceRect.angle > 90.0f) { + faceRect.angle -= 180.0f; + } + + // compute the smoothed face rect + if (_smoothedFaceRect.size.area() == 0) { + _smoothedFaceRect = faceRect; + + } else { + const float FACE_RECT_SMOOTHING = 0.9f; + _smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING); + _smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING); + _smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING); + _smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING); + _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); + } + + // use the face rect to compute the face transform, aspect ratio + Point2f sourcePoints[4]; + _smoothedFaceRect.points(sourcePoints); + Point2f destPoints[] = { Point2f(0, encodedHeight), Point2f(0, 0), Point2f(encodedWidth, 0) }; + faceTransform = getAffineTransform(sourcePoints, destPoints); + aspectRatio = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; + } + + const ushort ELEVEN_BIT_MINIMUM = 0; + const uchar EIGHT_BIT_MIDPOINT = 128; + double depthOffset; + if (!depth.empty()) { + if (_videoSendMode == FACE_VIDEO) { + // warp the face depth without interpolation (because it will contain invalid zero values) + _faceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + warpAffine(depth, _faceDepth, faceTransform, _faceDepth.size(), INTER_NEAREST); + + } else { + _faceDepth = depth; + } + _smoothedFaceDepth.create(encodedHeight, encodedWidth, CV_16UC1); + + // smooth the depth over time + const ushort ELEVEN_BIT_MAXIMUM = 2047; + const float DEPTH_SMOOTHING = 0.25f; + ushort* src = _faceDepth.ptr(); + ushort* dest = _smoothedFaceDepth.ptr(); + ushort minimumDepth = numeric_limits::max(); + for (int i = 0; i < encodedHeight; i++) { + for (int j = 0; j < encodedWidth; j++) { + ushort depth = *src++; + if (depth != ELEVEN_BIT_MINIMUM && depth != ELEVEN_BIT_MAXIMUM) { + minimumDepth = min(minimumDepth, depth); + *dest = (*dest == ELEVEN_BIT_MINIMUM) ? depth : (ushort)glm::mix(depth, *dest, DEPTH_SMOOTHING); + } + dest++; + } + } + const ushort MINIMUM_DEPTH_OFFSET = 64; + const float FIXED_MID_DEPTH = 640.0f; + float midFaceDepth = (_videoSendMode == FACE_VIDEO) ? (minimumDepth + MINIMUM_DEPTH_OFFSET) : FIXED_MID_DEPTH; + + // smooth the mid face depth over time + const float MID_FACE_DEPTH_SMOOTHING = 0.5f; + _smoothedMidFaceDepth = (_smoothedMidFaceDepth == UNINITIALIZED_FACE_DEPTH) ? midFaceDepth : + glm::mix(midFaceDepth, _smoothedMidFaceDepth, MID_FACE_DEPTH_SMOOTHING); // convert from 11 to 8 bits for preview/local display - const uchar EIGHT_BIT_MIDPOINT = 128; - double depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMeanFaceDepth; + depthOffset = EIGHT_BIT_MIDPOINT - _smoothedMidFaceDepth; depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, depthOffset); - - // likewise for the encoded representation + } + + QByteArray payload; + if (_videoSendMode != NO_VIDEO) { + if (_colorCodec.name == 0) { + // initialize encoder context(s) + vpx_codec_enc_cfg_t codecConfig; + vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0); + codecConfig.rc_target_bitrate = encodedWidth * encodedHeight * + codecConfig.rc_target_bitrate / codecConfig.g_w / codecConfig.g_h; + codecConfig.g_w = encodedWidth; + codecConfig.g_h = encodedHeight; + vpx_codec_enc_init(&_colorCodec, vpx_codec_vp8_cx(), &codecConfig, 0); + + if (!depth.empty()) { + codecConfig.rc_target_bitrate *= depthBitrateMultiplier; + vpx_codec_enc_init(&_depthCodec, vpx_codec_vp8_cx(), &codecConfig, 0); + } + } + + Mat transform; + if (_videoSendMode == FACE_VIDEO) { + // resize/rotate face into encoding rectangle + _faceColor.create(encodedHeight, encodedWidth, CV_8UC3); + warpAffine(color, _faceColor, faceTransform, _faceColor.size()); + + } else { + _faceColor = color; + } + + // convert from RGB to YV12: see http://www.fourcc.org/yuv.php and + // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor + const int ENCODED_BITS_PER_Y = 8; + const int ENCODED_BITS_PER_VU = 2; + const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU; + const int BITS_PER_BYTE = 8; + _encodedFace.resize(encodedWidth * encodedHeight * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE); + vpx_image_t vpxImage; + vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, encodedWidth, encodedHeight, 1, + (unsigned char*)_encodedFace.data()); uchar* yline = vpxImage.planes[0]; uchar* vline = vpxImage.planes[1]; uchar* uline = vpxImage.planes[2]; - const uchar EIGHT_BIT_MAXIMUM = 255; - for (int i = 0; i < ENCODED_FACE_HEIGHT; i += 2) { + const int Y_RED_WEIGHT = (int)(0.299 * 256); + const int Y_GREEN_WEIGHT = (int)(0.587 * 256); + const int Y_BLUE_WEIGHT = (int)(0.114 * 256); + const int V_RED_WEIGHT = (int)(0.713 * 256); + const int U_BLUE_WEIGHT = (int)(0.564 * 256); + int redIndex = 0; + int greenIndex = 1; + int blueIndex = 2; + if (format == GL_BGR) { + redIndex = 2; + blueIndex = 0; + } + for (int i = 0; i < encodedHeight; i += 2) { uchar* ydest = yline; uchar* vdest = vline; uchar* udest = uline; - for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { - ushort tl = *_faceDepth.ptr(i, j); - ushort tr = *_faceDepth.ptr(i, j + 1); - ushort bl = *_faceDepth.ptr(i + 1, j); - ushort br = *_faceDepth.ptr(i + 1, j + 1); - - uchar mask = EIGHT_BIT_MAXIMUM; + for (int j = 0; j < encodedWidth; j += 2) { + uchar* tl = _faceColor.ptr(i, j); + uchar* tr = _faceColor.ptr(i, j + 1); + uchar* bl = _faceColor.ptr(i + 1, j); + uchar* br = _faceColor.ptr(i + 1, j + 1); - ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(tl + depthOffset); - ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(tr + depthOffset); - ydest[vpxImage.stride[0]] = (bl == ELEVEN_BIT_MINIMUM) ? - (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(bl + depthOffset); - ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? - (mask = EIGHT_BIT_MIDPOINT) : saturate_cast(br + depthOffset); + ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[vpxImage.stride[0]] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] * + Y_GREEN_WEIGHT + bl[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[vpxImage.stride[0] + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] * + Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8; ydest += 2; - - *vdest++ = mask; - *udest++ = EIGHT_BIT_MIDPOINT; + + int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex]; + int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex]; + int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex]; + int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8; + + *vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128; + *udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128; } yline += vpxImage.stride[0] * 2; vline += vpxImage.stride[1]; @@ -681,24 +674,79 @@ void FrameGrabber::grabFrame() { } // encode the frame - vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME); + vpx_codec_encode(&_colorCodec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); + + // start the payload off with the aspect ratio (zero for full frame) + payload.append((const char*)&aspectRatio, sizeof(float)); // extract the encoded frame vpx_codec_iter_t iterator = 0; const vpx_codec_cx_pkt_t* packet; - while ((packet = vpx_codec_get_cx_data(&_depthCodec, &iterator)) != 0) { + while ((packet = vpx_codec_get_cx_data(&_colorCodec, &iterator)) != 0) { if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { + // prepend the length, which will indicate whether there's a depth frame too + payload.append((const char*)&packet->data.frame.sz, sizeof(packet->data.frame.sz)); payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); } } + + if (!depth.empty()) { + // convert with mask + uchar* yline = vpxImage.planes[0]; + uchar* vline = vpxImage.planes[1]; + uchar* uline = vpxImage.planes[2]; + const uchar EIGHT_BIT_MAXIMUM = 255; + for (int i = 0; i < encodedHeight; i += 2) { + uchar* ydest = yline; + uchar* vdest = vline; + uchar* udest = uline; + for (int j = 0; j < encodedWidth; j += 2) { + ushort tl = *_smoothedFaceDepth.ptr(i, j); + ushort tr = *_smoothedFaceDepth.ptr(i, j + 1); + ushort bl = *_smoothedFaceDepth.ptr(i + 1, j); + ushort br = *_smoothedFaceDepth.ptr(i + 1, j + 1); + + uchar mask = EIGHT_BIT_MAXIMUM; + + ydest[0] = (tl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(tl + depthOffset); + ydest[1] = (tr == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(tr + depthOffset); + ydest[vpxImage.stride[0]] = (bl == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(bl + depthOffset); + ydest[vpxImage.stride[0] + 1] = (br == ELEVEN_BIT_MINIMUM) ? (mask = EIGHT_BIT_MIDPOINT) : + saturate_cast(br + depthOffset); + ydest += 2; + + *vdest++ = mask; + *udest++ = EIGHT_BIT_MIDPOINT; + } + yline += vpxImage.stride[0] * 2; + vline += vpxImage.stride[1]; + uline += vpxImage.stride[2]; + } + + // encode the frame + vpx_codec_encode(&_depthCodec, &vpxImage, _frameCount, 1, 0, VPX_DL_REALTIME); + + // extract the encoded frame + vpx_codec_iter_t iterator = 0; + const vpx_codec_cx_pkt_t* packet; + while ((packet = vpx_codec_get_cx_data(&_depthCodec, &iterator)) != 0) { + if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { + payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); + } + } + } } QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", - Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); - + Q_ARG(int, _frameCount), Q_ARG(QByteArray, payload)); + QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", - Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMeanFaceDepth), - Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); + Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), Q_ARG(float, _smoothedMidFaceDepth), + Q_ARG(float, aspectRatio), Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(bool, !payload.isEmpty()), + Q_ARG(JointVector, joints)); } bool FrameGrabber::init() { @@ -767,6 +815,17 @@ void FrameGrabber::updateHSVFrame(const Mat& frame, int format) { inRange(_hsvFrame, Scalar(0, 55, 65), Scalar(180, 256, 256), _mask); } +void FrameGrabber::destroyCodecs() { + if (_colorCodec.name != 0) { + vpx_codec_destroy(&_colorCodec); + _colorCodec.name = 0; + } + if (_depthCodec.name != 0) { + vpx_codec_destroy(&_depthCodec); + _depthCodec.name = 0; + } +} + Joint::Joint(const glm::vec3& position, const glm::quat& rotation, const glm::vec3& projected) : isValid(true), position(position), rotation(rotation), projected(projected) { } diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index 3910bb4a19..35c85cc63a 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -44,12 +44,18 @@ public: Webcam(); ~Webcam(); + FrameGrabber* getGrabber() { return _grabber; } + bool isActive() const { return _active; } + + bool isSending() const { return _sending; } GLuint getColorTextureID() const { return _colorTextureID; } GLuint getDepthTextureID() const { return _depthTextureID; } const cv::Size2f& getTextureSize() const { return _textureSize; } + float getAspectRatio() const { return _aspectRatio; } + const cv::RotatedRect& getFaceRect() const { return _faceRect; } const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; } @@ -57,13 +63,14 @@ public: const JointVector& getEstimatedJoints() const { return _estimatedJoints; } void reset(); - void renderPreview(int screenWidth, int screenHeight); + void renderPreview(int screenWidth, int screenHeight); public slots: void setEnabled(bool enabled); - void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float meanFaceDepth, - const cv::RotatedRect& faceRect, const JointVector& joints); + void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, float midFaceDepth, + float aspectRatio, const cv::RotatedRect& faceRect, bool sending, const JointVector& joints); + void setSkeletonTrackingOn(bool toggle) { _skeletonTrackingOn = toggle; }; private: @@ -72,9 +79,11 @@ private: bool _enabled; bool _active; + bool _sending; GLuint _colorTextureID; GLuint _depthTextureID; cv::Size2f _textureSize; + float _aspectRatio; cv::RotatedRect _faceRect; cv::RotatedRect _initialFaceRect; float _initialFaceDepth; @@ -88,6 +97,8 @@ private: glm::vec3 _estimatedPosition; glm::vec3 _estimatedRotation; JointVector _estimatedJoints; + + bool _skeletonTrackingOn; }; class FrameGrabber : public QObject { @@ -100,16 +111,21 @@ public: public slots: + void cycleVideoSendMode(); void reset(); void shutdown(); void grabFrame(); private: + enum VideoSendMode { NO_VIDEO, FACE_VIDEO, FULL_FRAME_VIDEO, VIDEO_SEND_MODE_COUNT }; + bool init(); void updateHSVFrame(const cv::Mat& frame, int format); + void destroyCodecs(); bool _initialized; + VideoSendMode _videoSendMode; CvCapture* _capture; cv::CascadeClassifier _faceCascade; cv::Mat _hsvFrame; @@ -118,13 +134,14 @@ private: cv::Mat _backProject; cv::Rect _searchWindow; cv::Mat _grayDepthFrame; - float _smoothedMeanFaceDepth; + float _smoothedMidFaceDepth; vpx_codec_ctx_t _colorCodec; vpx_codec_ctx_t _depthCodec; int _frameCount; cv::Mat _faceColor; cv::Mat _faceDepth; + cv::Mat _smoothedFaceDepth; QByteArray _encodedFace; cv::RotatedRect _smoothedFaceRect; diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 1af82083ae..6807e521b8 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -314,10 +314,7 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook, estimatedPosition = webcam->getEstimatedPosition(); // apply face data - _head.getFace().setColorTextureID(webcam->getColorTextureID()); - _head.getFace().setDepthTextureID(webcam->getDepthTextureID()); - _head.getFace().setTextureSize(webcam->getTextureSize()); - _head.getFace().setTextureRect(webcam->getFaceRect()); + _head.getFace().setFrameFromWebcam(); // compute and store the joint rotations const JointVector& joints = webcam->getEstimatedJoints(); @@ -334,7 +331,7 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook, } } } else { - _head.getFace().setColorTextureID(0); + _head.getFace().clearFrame(); } _head.setPitch(estimatedRotation.x * amplifyAngle.x + pitchFromTouch); _head.setYaw(estimatedRotation.y * amplifyAngle.y + yawFromTouch); @@ -892,21 +889,15 @@ void Avatar::updateCollisionWithSphere(glm::vec3 position, float radius, float d } void Avatar::updateCollisionWithEnvironment(float deltaTime) { - glm::vec3 up = getBodyUpDirection(); float radius = _height * 0.125f; const float ENVIRONMENT_SURFACE_ELASTICITY = 1.0f; const float ENVIRONMENT_SURFACE_DAMPING = 0.01; const float ENVIRONMENT_COLLISION_FREQUENCY = 0.05f; - const float VISIBLE_GROUND_COLLISION_VELOCITY = 0.2f; glm::vec3 penetration; if (Application::getInstance()->getEnvironment()->findCapsulePenetration( _position - up * (_pelvisFloatingHeight - radius), _position + up * (_height - _pelvisFloatingHeight - radius), radius, penetration)) { - float velocityTowardCollision = glm::dot(_velocity, glm::normalize(penetration)); - if (velocityTowardCollision > VISIBLE_GROUND_COLLISION_VELOCITY) { - Application::getInstance()->setGroundPlaneImpact(1.0f); - } _lastCollisionPosition = _position; updateCollisionSound(penetration, deltaTime, ENVIRONMENT_COLLISION_FREQUENCY); applyHardCollision(penetration, ENVIRONMENT_SURFACE_ELASTICITY, ENVIRONMENT_SURFACE_DAMPING); @@ -1300,9 +1291,15 @@ float Avatar::getBallRenderAlpha(int ball, bool lookingInMirror) const { } void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { - - // Render the body as balls and cones - if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) { + + if (_head.getFace().isFullFrame()) { + // Render the full-frame video + float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); + if (alpha > 0.0f) { + _head.getFace().render(1.0f); + } + } else if (renderAvatarBalls || !_voxels.getVoxelURL().isValid()) { + // Render the body as balls and cones for (int b = 0; b < NUM_AVATAR_BODY_BALLS; b++) { float alpha = getBallRenderAlpha(b, lookingInMirror); diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp index ff31241c54..0541b3c21c 100644 --- a/interface/src/avatar/Face.cpp +++ b/interface/src/avatar/Face.cpp @@ -17,6 +17,7 @@ #include "Avatar.h" #include "Head.h" #include "Face.h" +#include "Webcam.h" #include "renderer/ProgramObject.h" using namespace cv; @@ -25,7 +26,6 @@ ProgramObject* Face::_program = 0; int Face::_texCoordCornerLocation; int Face::_texCoordRightLocation; int Face::_texCoordUpLocation; -int Face::_aspectRatioLocation; GLuint Face::_vboID; GLuint Face::_iboID; @@ -55,17 +55,25 @@ Face::~Face() { } } -void Face::setTextureRect(const cv::RotatedRect& textureRect) { - _textureRect = textureRect; - _aspectRatio = _textureRect.size.width / _textureRect.size.height; +void Face::setFrameFromWebcam() { + Webcam* webcam = Application::getInstance()->getWebcam(); + if (webcam->isSending()) { + _colorTextureID = webcam->getColorTextureID(); + _depthTextureID = webcam->getDepthTextureID(); + _textureSize = webcam->getTextureSize(); + _textureRect = webcam->getFaceRect(); + _aspectRatio = webcam->getAspectRatio(); + + } else { + clearFrame(); + } } +void Face::clearFrame() { + _colorTextureID = 0; +} + int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { - if (_colorCodec.name == 0) { - // initialize decoder context - vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0); - } - // skip the header unsigned char* packetPosition = packetData; int frameCount = *(uint32_t*)packetPosition; @@ -89,110 +97,135 @@ int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { int payloadSize = dataBytes - (packetPosition - packetData); memcpy(_arrivingFrame.data() + frameOffset, packetPosition, payloadSize); - if ((_frameBytesRemaining -= payloadSize) <= 0) { - float aspectRatio = *(const float*)_arrivingFrame.constData(); - size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float)); - const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t)); - vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0); - vpx_codec_iter_t iterator = 0; - vpx_image_t* image; - while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { - // convert from YV12 to RGB - Mat color(image->d_h, image->d_w, CV_8UC3); - uchar* yline = image->planes[0]; - uchar* vline = image->planes[1]; - uchar* uline = image->planes[2]; - const int RED_V_WEIGHT = (int)(1.403 * 256); - const int GREEN_V_WEIGHT = (int)(0.714 * 256); - const int GREEN_U_WEIGHT = (int)(0.344 * 256); - const int BLUE_U_WEIGHT = (int)(1.773 * 256); - for (int i = 0; i < image->d_h; i += 2) { - uchar* ysrc = yline; - uchar* vsrc = vline; - uchar* usrc = uline; - for (int j = 0; j < image->d_w; j += 2) { - uchar* tl = color.ptr(i, j); - uchar* tr = color.ptr(i, j + 1); - uchar* bl = color.ptr(i + 1, j); - uchar* br = color.ptr(i + 1, j + 1); - - int v = *vsrc++ - 128; - int u = *usrc++ - 128; - - int redOffset = (RED_V_WEIGHT * v) >> 8; - int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; - int blueOffset = (BLUE_U_WEIGHT * u) >> 8; - - int ytl = ysrc[0]; - int ytr = ysrc[1]; - int ybl = ysrc[image->w]; - int ybr = ysrc[image->w + 1]; - ysrc += 2; - - tl[0] = ytl + redOffset; - tl[1] = ytl - greenOffset; - tl[2] = ytl + blueOffset; - - tr[0] = ytr + redOffset; - tr[1] = ytr - greenOffset; - tr[2] = ytr + blueOffset; - - bl[0] = ybl + redOffset; - bl[1] = ybl - greenOffset; - bl[2] = ybl + blueOffset; - - br[0] = ybr + redOffset; - br[1] = ybr - greenOffset; - br[2] = ybr + blueOffset; - } - yline += image->stride[0] * 2; - vline += image->stride[1]; - uline += image->stride[2]; + if ((_frameBytesRemaining -= payloadSize) > 0) { + return dataBytes; // wait for the rest of the frame + } + + if (frameSize == 0) { + // destroy the codecs, if we have any + destroyCodecs(); + + // disables video data + QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, Mat()), + Q_ARG(cv::Mat, Mat()), Q_ARG(float, 0.0f)); + return dataBytes; + } + + // the switch from full frame to not (or vice versa) requires us to reinit the codecs + float aspectRatio = *(const float*)_arrivingFrame.constData(); + bool fullFrame = (aspectRatio == FULL_FRAME_ASPECT); + if (fullFrame != _lastFullFrame) { + destroyCodecs(); + _lastFullFrame = fullFrame; + } + + if (_colorCodec.name == 0) { + // initialize decoder context + vpx_codec_dec_init(&_colorCodec, vpx_codec_vp8_dx(), 0, 0); + } + + size_t colorSize = *(const size_t*)(_arrivingFrame.constData() + sizeof(float)); + const uint8_t* colorData = (const uint8_t*)(_arrivingFrame.constData() + sizeof(float) + sizeof(size_t)); + vpx_codec_decode(&_colorCodec, colorData, colorSize, 0, 0); + vpx_codec_iter_t iterator = 0; + vpx_image_t* image; + while ((image = vpx_codec_get_frame(&_colorCodec, &iterator)) != 0) { + // convert from YV12 to RGB: see http://www.fourcc.org/yuv.php and + // http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor + Mat color(image->d_h, image->d_w, CV_8UC3); + uchar* yline = image->planes[0]; + uchar* vline = image->planes[1]; + uchar* uline = image->planes[2]; + const int RED_V_WEIGHT = (int)(1.403 * 256); + const int GREEN_V_WEIGHT = (int)(0.714 * 256); + const int GREEN_U_WEIGHT = (int)(0.344 * 256); + const int BLUE_U_WEIGHT = (int)(1.773 * 256); + for (int i = 0; i < image->d_h; i += 2) { + uchar* ysrc = yline; + uchar* vsrc = vline; + uchar* usrc = uline; + for (int j = 0; j < image->d_w; j += 2) { + uchar* tl = color.ptr(i, j); + uchar* tr = color.ptr(i, j + 1); + uchar* bl = color.ptr(i + 1, j); + uchar* br = color.ptr(i + 1, j + 1); + + int v = *vsrc++ - 128; + int u = *usrc++ - 128; + + int redOffset = (RED_V_WEIGHT * v) >> 8; + int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; + int blueOffset = (BLUE_U_WEIGHT * u) >> 8; + + int ytl = ysrc[0]; + int ytr = ysrc[1]; + int ybl = ysrc[image->w]; + int ybr = ysrc[image->w + 1]; + ysrc += 2; + + tl[0] = ytl + redOffset; + tl[1] = ytl - greenOffset; + tl[2] = ytl + blueOffset; + + tr[0] = ytr + redOffset; + tr[1] = ytr - greenOffset; + tr[2] = ytr + blueOffset; + + bl[0] = ybl + redOffset; + bl[1] = ybl - greenOffset; + bl[2] = ybl + blueOffset; + + br[0] = ybr + redOffset; + br[1] = ybr - greenOffset; + br[2] = ybr + blueOffset; } - Mat depth; - - const uint8_t* depthData = colorData + colorSize; - int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData()); - if (depthSize > 0) { - if (_depthCodec.name == 0) { - // initialize decoder context - vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0); - } - vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0); - vpx_codec_iter_t iterator = 0; - vpx_image_t* image; - while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) { - depth.create(image->d_h, image->d_w, CV_8UC1); - uchar* yline = image->planes[0]; - uchar* vline = image->planes[1]; - const uchar EIGHT_BIT_MAXIMUM = 255; - const uchar MASK_THRESHOLD = 192; - for (int i = 0; i < image->d_h; i += 2) { - uchar* ysrc = yline; - uchar* vsrc = vline; - for (int j = 0; j < image->d_w; j += 2) { - if (*vsrc++ < MASK_THRESHOLD) { - *depth.ptr(i, j) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM; - *depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM; - - } else { - *depth.ptr(i, j) = ysrc[0]; - *depth.ptr(i, j + 1) = ysrc[1]; - *depth.ptr(i + 1, j) = ysrc[image->stride[0]]; - *depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1]; - } - ysrc += 2; - } - yline += image->stride[0] * 2; - vline += image->stride[1]; - } - } - } - QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color), - Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio)); + yline += image->stride[0] * 2; + vline += image->stride[1]; + uline += image->stride[2]; } + Mat depth; + + const uint8_t* depthData = colorData + colorSize; + int depthSize = _arrivingFrame.size() - ((const char*)depthData - _arrivingFrame.constData()); + if (depthSize > 0) { + if (_depthCodec.name == 0) { + // initialize decoder context + vpx_codec_dec_init(&_depthCodec, vpx_codec_vp8_dx(), 0, 0); + } + vpx_codec_decode(&_depthCodec, depthData, depthSize, 0, 0); + vpx_codec_iter_t iterator = 0; + vpx_image_t* image; + while ((image = vpx_codec_get_frame(&_depthCodec, &iterator)) != 0) { + depth.create(image->d_h, image->d_w, CV_8UC1); + uchar* yline = image->planes[0]; + uchar* vline = image->planes[1]; + const uchar EIGHT_BIT_MAXIMUM = 255; + const uchar MASK_THRESHOLD = 192; + for (int i = 0; i < image->d_h; i += 2) { + uchar* ysrc = yline; + uchar* vsrc = vline; + for (int j = 0; j < image->d_w; j += 2) { + if (*vsrc++ < MASK_THRESHOLD) { + *depth.ptr(i, j) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i, j + 1) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i + 1, j) = EIGHT_BIT_MAXIMUM; + *depth.ptr(i + 1, j + 1) = EIGHT_BIT_MAXIMUM; + + } else { + *depth.ptr(i, j) = ysrc[0]; + *depth.ptr(i, j + 1) = ysrc[1]; + *depth.ptr(i + 1, j) = ysrc[image->stride[0]]; + *depth.ptr(i + 1, j + 1) = ysrc[image->stride[0] + 1]; + } + ysrc += 2; + } + yline += image->stride[0] * 2; + vline += image->stride[1]; + } + } + } + QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color), + Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio)); } return dataBytes; @@ -208,9 +241,22 @@ bool Face::render(float alpha) { glm::quat orientation = _owningHead->getOrientation(); glm::vec3 axis = glm::axis(orientation); glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); - float scale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); - glScalef(scale, scale, scale); - + + float aspect, xScale, zScale; + if (_aspectRatio == FULL_FRAME_ASPECT) { + aspect = _textureSize.width / _textureSize.height; + const float FULL_FRAME_SCALE = 0.5f; + xScale = FULL_FRAME_SCALE * _owningHead->getScale(); + zScale = xScale * 0.3f; + + } else { + aspect = _aspectRatio; + xScale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + zScale = xScale * 1.5f; + glTranslatef(0.0f, -xScale * 0.75f, -xScale); + } + glScalef(xScale, xScale / aspect, zScale); + glColor4f(1.0f, 1.0f, 1.0f, alpha); Point2f points[4]; @@ -243,7 +289,6 @@ bool Face::render(float alpha) { _texCoordCornerLocation = _program->uniformLocation("texCoordCorner"); _texCoordRightLocation = _program->uniformLocation("texCoordRight"); _texCoordUpLocation = _program->uniformLocation("texCoordUp"); - _aspectRatioLocation = _program->uniformLocation("aspectRatio"); glGenBuffers(1, &_vboID); glBindBuffer(GL_ARRAY_BUFFER, _vboID); @@ -292,7 +337,6 @@ bool Face::render(float alpha) { (points[3].x - points[0].x) / _textureSize.width, (points[3].y - points[0].y) / _textureSize.height); _program->setUniformValue(_texCoordUpLocation, (points[1].x - points[0].x) / _textureSize.width, (points[1].y - points[0].y) / _textureSize.height); - _program->setUniformValue(_aspectRatioLocation, _aspectRatio); glEnableClientState(GL_VERTEX_ARRAY); glVertexPointer(2, GL_FLOAT, 0, 0); @@ -324,13 +368,13 @@ bool Face::render(float alpha) { glBegin(GL_QUADS); glTexCoord2f(points[0].x / _textureSize.width, points[0].y / _textureSize.height); - glVertex3f(0.5f, -0.5f / _aspectRatio, -0.5f); + glVertex3f(0.5f, -0.5f, 0.0f); glTexCoord2f(points[1].x / _textureSize.width, points[1].y / _textureSize.height); - glVertex3f(0.5f, 0.5f / _aspectRatio, -0.5f); + glVertex3f(0.5f, 0.5f, 0.0f); glTexCoord2f(points[2].x / _textureSize.width, points[2].y / _textureSize.height); - glVertex3f(-0.5f, 0.5f / _aspectRatio, -0.5f); + glVertex3f(-0.5f, 0.5f, 0.0f); glTexCoord2f(points[3].x / _textureSize.width, points[3].y / _textureSize.height); - glVertex3f(-0.5f, -0.5f / _aspectRatio, -0.5f); + glVertex3f(-0.5f, -0.5f, 0.0f); glEnd(); glDisable(GL_TEXTURE_2D); @@ -348,23 +392,40 @@ void Face::cycleRenderMode() { } void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) { + if (color.empty()) { + // release our textures, if any; there's no more video + if (_colorTextureID != 0) { + glDeleteTextures(1, &_colorTextureID); + _colorTextureID = 0; + } + if (_depthTextureID != 0) { + glDeleteTextures(1, &_depthTextureID); + _depthTextureID = 0; + } + return; + } + if (_colorTextureID == 0) { glGenTextures(1, &_colorTextureID); - glBindTexture(GL_TEXTURE_2D, _colorTextureID); + } + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + bool recreateTextures = (_textureSize.width != color.cols || _textureSize.height != color.rows); + if (recreateTextures) { glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, color.cols, color.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); _textureSize = color.size(); _textureRect = RotatedRect(Point2f(color.cols * 0.5f, color.rows * 0.5f), _textureSize, 0.0f); - + } else { - glBindTexture(GL_TEXTURE_2D, _colorTextureID); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, color.cols, color.rows, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); } if (!depth.empty()) { if (_depthTextureID == 0) { glGenTextures(1, &_depthTextureID); - glBindTexture(GL_TEXTURE_2D, _depthTextureID); + } + glBindTexture(GL_TEXTURE_2D, _depthTextureID); + if (recreateTextures) { glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depth.cols, depth.rows, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); @@ -380,3 +441,13 @@ void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRati _aspectRatio = aspectRatio; } +void Face::destroyCodecs() { + if (_colorCodec.name != 0) { + vpx_codec_destroy(&_colorCodec); + _colorCodec.name = 0; + } + if (_depthCodec.name != 0) { + vpx_codec_destroy(&_depthCodec); + _depthCodec.name = 0; + } +} diff --git a/interface/src/avatar/Face.h b/interface/src/avatar/Face.h index d4812fecfb..893318f186 100644 --- a/interface/src/avatar/Face.h +++ b/interface/src/avatar/Face.h @@ -20,6 +20,8 @@ class Head; class ProgramObject; +const float FULL_FRAME_ASPECT = 0.0f; + class Face : public QObject { Q_OBJECT @@ -28,10 +30,10 @@ public: Face(Head* owningHead); ~Face(); - void setColorTextureID(GLuint colorTextureID) { _colorTextureID = colorTextureID; } - void setDepthTextureID(GLuint depthTextureID) { _depthTextureID = depthTextureID; } - void setTextureSize(const cv::Size2f& textureSize) { _textureSize = textureSize; } - void setTextureRect(const cv::RotatedRect& textureRect); + bool isFullFrame() const { return _colorTextureID != 0 && _aspectRatio == FULL_FRAME_ASPECT; } + + void setFrameFromWebcam(); + void clearFrame(); int processVideoMessage(unsigned char* packetData, size_t dataBytes); @@ -49,6 +51,8 @@ private: enum RenderMode { MESH, POINTS, RENDER_MODE_COUNT }; + void destroyCodecs(); + Head* _owningHead; RenderMode _renderMode; GLuint _colorTextureID; @@ -59,6 +63,7 @@ private: vpx_codec_ctx_t _colorCodec; vpx_codec_ctx_t _depthCodec; + bool _lastFullFrame; QByteArray _arrivingFrame; int _frameCount; @@ -68,7 +73,6 @@ private: static int _texCoordCornerLocation; static int _texCoordRightLocation; static int _texCoordUpLocation; - static int _aspectRatioLocation; static GLuint _vboID; static GLuint _iboID; }; diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 28b1af0603..45a2117285 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -51,6 +51,7 @@ void Hand::reset() { void Hand::simulate(float deltaTime, bool isMine) { + if (_isRaveGloveActive) { updateRaveGloveParticles(deltaTime); } @@ -63,7 +64,8 @@ void Hand::calculateGeometry() { _basePosition = head.getPosition() + head.getOrientation() * offset; _baseOrientation = head.getOrientation(); - _leapBalls.clear(); + // generate finger tip balls.... + _leapFingerTipBalls.clear(); for (size_t i = 0; i < getNumPalms(); ++i) { PalmData& palm = getPalms()[i]; if (palm.isActive()) { @@ -71,8 +73,8 @@ void Hand::calculateGeometry() { FingerData& finger = palm.getFingers()[f]; if (finger.isActive()) { const float standardBallRadius = 0.01f; - _leapBalls.resize(_leapBalls.size() + 1); - HandBall& ball = _leapBalls.back(); + _leapFingerTipBalls.resize(_leapFingerTipBalls.size() + 1); + HandBall& ball = _leapFingerTipBalls.back(); ball.rotation = _baseOrientation; ball.position = finger.getTipPosition(); ball.radius = standardBallRadius; @@ -82,6 +84,27 @@ void Hand::calculateGeometry() { } } } + + // generate finger rot balls.... + _leapFingerRootBalls.clear(); + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& palm = getPalms()[i]; + if (palm.isActive()) { + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + if (finger.isActive()) { + const float standardBallRadius = 0.01f; + _leapFingerRootBalls.resize(_leapFingerRootBalls.size() + 1); + HandBall& ball = _leapFingerRootBalls.back(); + ball.rotation = _baseOrientation; + ball.position = finger.getRootPosition(); + ball.radius = standardBallRadius; + ball.touchForce = 0.0; + ball.isCollidable = true; + } + } + } + } } void Hand::setRaveGloveEffectsMode(QKeyEvent* event) { @@ -120,8 +143,9 @@ void Hand::render(bool lookingInMirror) { glEnable(GL_RESCALE_NORMAL); if ( SHOW_LEAP_HAND ) { - renderFingerTrails(); - renderHandSpheres(); + //renderLeapHands(); + renderLeapFingerTrails(); + renderLeapHandSpheres(); } } @@ -153,18 +177,64 @@ void Hand::renderRaveGloveStage() { } } -void Hand::renderHandSpheres() { + +void Hand::renderLeapHands() { + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& hand = getPalms()[i]; + if (hand.isActive()) { + renderLeapHand(hand); + } + } +} + +void Hand::renderLeapHand(PalmData& hand) { + + glPushMatrix(); + const float palmThickness = 0.002f; + glColor4f(0.5f, 0.5f, 0.5f, 1.0); + glm::vec3 tip = hand.getPosition(); + glm::vec3 root = hand.getPosition() + hand.getNormal() * palmThickness; + Avatar::renderJointConnectingCone(root, tip, 0.05, 0.03); + + for (size_t f = 0; f < hand.getNumFingers(); ++f) { + FingerData& finger = hand.getFingers()[f]; + if (finger.isActive()) { + glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, 0.5); + glm::vec3 tip = finger.getTipPosition(); + glm::vec3 root = finger.getRootPosition(); + Avatar::renderJointConnectingCone(root, tip, 0.001, 0.003); + } + } + + glPopMatrix(); +} + + +void Hand::renderLeapHandSpheres() { glPushMatrix(); // Draw the leap balls - for (size_t i = 0; i < _leapBalls.size(); i++) { + for (size_t i = 0; i < _leapFingerTipBalls.size(); i++) { float alpha = 1.0f; if (alpha > 0.0f) { glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, alpha); glPushMatrix(); - glTranslatef(_leapBalls[i].position.x, _leapBalls[i].position.y, _leapBalls[i].position.z); - glutSolidSphere(_leapBalls[i].radius, 20.0f, 20.0f); + glTranslatef(_leapFingerTipBalls[i].position.x, _leapFingerTipBalls[i].position.y, _leapFingerTipBalls[i].position.z); + glutSolidSphere(_leapFingerTipBalls[i].radius, 20.0f, 20.0f); + glPopMatrix(); + } + } + + for (size_t i = 0; i < _leapFingerRootBalls.size(); i++) { + float alpha = 1.0f; + + if (alpha > 0.0f) { + glColor4f(0.3f, 0.4f, 0.6f, alpha); + + glPushMatrix(); + glTranslatef(_leapFingerRootBalls[i].position.x, _leapFingerRootBalls[i].position.y, _leapFingerRootBalls[i].position.z); + glutSolidSphere(_leapFingerRootBalls[i].radius, 20.0f, 20.0f); glPopMatrix(); } } @@ -200,7 +270,7 @@ void Hand::renderHandSpheres() { glPopMatrix(); } -void Hand::renderFingerTrails() { +void Hand::renderLeapFingerTrails() { // Draw the finger root cones for (size_t i = 0; i < getNumPalms(); ++i) { PalmData& palm = getPalms()[i]; @@ -229,6 +299,7 @@ void Hand::renderFingerTrails() { } } + void Hand::setLeapHands(const std::vector& handPositions, const std::vector& handNormals) { for (size_t i = 0; i < getNumPalms(); ++i) { @@ -244,69 +315,28 @@ void Hand::setLeapHands(const std::vector& handPositions, } } -// call this right after the geometry of the leap hands are set + +// call this soon after the geometry of the leap hands are set void Hand::updateRaveGloveEmitters() { - bool debug = false; + for (size_t i = 0; i < NUM_FINGERS; i++) { + _raveGloveParticleSystem.setEmitterActive(_raveGloveEmitter[i], false); + } - if (_raveGloveInitialized) { - - if(debug) printf( "\n" ); - if(debug) printf( "------------------------------------\n" ); - if(debug) printf( "updating rave glove emitters:\n" ); - if(debug) printf( "------------------------------------\n" ); - - int emitterIndex = 0; - - for (size_t i = 0; i < getNumPalms(); ++i) { - PalmData& palm = getPalms()[i]; + for (size_t i = 0; i < _leapFingerTipBalls.size(); i++) { + if (i < NUM_FINGERS) { + glm::vec3 fingerDirection = _leapFingerTipBalls[i].position - _leapFingerRootBalls[i].position; + float fingerLength = glm::length(fingerDirection); - if(debug) printf( "\n" ); - if(debug) printf( "palm %d ", (int)i ); - - if (palm.isActive()) { - - if(debug) printf( "is active\n" ); - - for (size_t f = 0; f < palm.getNumFingers(); ++f) { - FingerData& finger = palm.getFingers()[f]; - - if(debug) printf( "emitterIndex %d: ", emitterIndex ); - - if (finger.isActive()) { - - if ((emitterIndex >=0) - && (emitterIndex < NUM_FINGERS)) { - - assert(emitterIndex >=0 ); - assert(emitterIndex < NUM_FINGERS ); - - if(debug) printf( "_raveGloveEmitter[%d] = %d\n", emitterIndex, _raveGloveEmitter[emitterIndex] ); - - glm::vec3 fingerDirection = finger.getTipPosition() - finger.getRootPosition(); - float fingerLength = glm::length(fingerDirection); - - if (fingerLength > 0.0f) { - fingerDirection /= fingerLength; - } else { - fingerDirection = IDENTITY_UP; - } - - assert(_raveGloveEmitter[emitterIndex] >=0 ); - assert(_raveGloveEmitter[emitterIndex] < NUM_FINGERS ); - - _raveGloveParticleSystem.setEmitterPosition (_raveGloveEmitter[emitterIndex], finger.getTipPosition()); - _raveGloveParticleSystem.setEmitterDirection(_raveGloveEmitter[emitterIndex], fingerDirection); - } - } else { - if(debug) printf( "BOGUS finger\n" ); - } - - emitterIndex ++; - } + if (fingerLength > 0.0f) { + fingerDirection /= fingerLength; } else { - if(debug) printf( "is NOT active\n" ); + fingerDirection = IDENTITY_UP; } + + _raveGloveParticleSystem.setEmitterActive (_raveGloveEmitter[i], true); + _raveGloveParticleSystem.setEmitterPosition (_raveGloveEmitter[i], _leapFingerTipBalls[i].position); + _raveGloveParticleSystem.setEmitterDirection(_raveGloveEmitter[i], fingerDirection); } } } @@ -317,16 +347,11 @@ void Hand::updateRaveGloveParticles(float deltaTime) { if (!_raveGloveInitialized) { - //printf( "Initializing rave glove emitters:\n" ); - //printf( "The indices of the emitters are:\n" ); - // start up the rave glove finger particles... for ( int f = 0; f< NUM_FINGERS; f ++ ) { - _raveGloveEmitter[f] = _raveGloveParticleSystem.addEmitter(); + _raveGloveEmitter[f] = _raveGloveParticleSystem.addEmitter(); assert( _raveGloveEmitter[f] >= 0 ); assert( _raveGloveEmitter[f] != NULL_EMITTER ); - - //printf( "%d\n", _raveGloveEmitter[f] ); } setRaveGloveMode(RAVE_GLOVE_EFFECTS_MODE_FIRE); @@ -339,13 +364,13 @@ void Hand::updateRaveGloveParticles(float deltaTime) { // this rave glove effect oscillates though various colors and radii that are meant to show off some effects if (_raveGloveMode == RAVE_GLOVE_EFFECTS_MODE_THROBBING_COLOR) { ParticleSystem::ParticleAttributes attributes; - float red = 0.5f + 0.5f * sinf(_raveGloveClock * 1.4f); - float green = 0.5f + 0.5f * cosf(_raveGloveClock * 1.7f); - float blue = 0.5f + 0.5f * sinf(_raveGloveClock * 2.0f); + float red = 0.5f + 0.5f * sinf(_raveGloveClock * 2.4f); + float green = 0.5f + 0.5f * cosf(_raveGloveClock * 2.7f); + float blue = 0.5f + 0.5f * sinf(_raveGloveClock * 3.0f); float alpha = 1.0f; attributes.color = glm::vec4(red, green, blue, alpha); - attributes.radius = 0.01f + 0.005f * sinf(_raveGloveClock * 2.2f); + attributes.radius = 0.01f + 0.003f * sinf(_raveGloveClock * 50.0f); attributes.modulationAmplitude = 0.0f; for ( int f = 0; f< NUM_FINGERS; f ++ ) { @@ -360,6 +385,8 @@ void Hand::updateRaveGloveParticles(float deltaTime) { } } + + void Hand::setRaveGloveMode(int mode) { _raveGloveMode = mode; @@ -376,7 +403,7 @@ void Hand::setRaveGloveMode(int mode) { if (mode == RAVE_GLOVE_EFFECTS_MODE_THROBBING_COLOR) { _raveGloveParticleSystem.setParticleRenderStyle (_raveGloveEmitter[f], PARTICLE_RENDER_STYLE_SPHERE ); _raveGloveParticleSystem.setShowingEmitterBaseParticle(_raveGloveEmitter[f], true ); - _raveGloveParticleSystem.setEmitterParticleLifespan (_raveGloveEmitter[f], 0.0f ); + _raveGloveParticleSystem.setEmitterParticleLifespan (_raveGloveEmitter[f], 0.03f ); _raveGloveParticleSystem.setEmitterThrust (_raveGloveEmitter[f], 0.0f ); _raveGloveParticleSystem.setEmitterRate (_raveGloveEmitter[f], 30.0f ); _raveGloveParticleSystem.setEmitterParticleResolution (_raveGloveEmitter[f], 20 ); @@ -650,7 +677,7 @@ void Hand::setRaveGloveMode(int mode) { _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes); //----------------------------------------- - // throb + // long sparkler //----------------------------------------- } else if (mode == RAVE_GLOVE_EFFECTS_MODE_LONG_SPARKLER) { @@ -672,6 +699,30 @@ void Hand::setRaveGloveMode(int mode) { _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_1, attributes); _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_2, attributes); _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes); + + //----------------------------------------- + // throb + //----------------------------------------- + } else if (mode == RAVE_GLOVE_EFFECTS_MODE_THROB) { + + _raveGloveParticleSystem.setParticleRenderStyle (_raveGloveEmitter[f], PARTICLE_RENDER_STYLE_SPHERE ); + _raveGloveParticleSystem.setShowingEmitterBaseParticle(_raveGloveEmitter[f], true ); + _raveGloveParticleSystem.setEmitterParticleLifespan (_raveGloveEmitter[f], 0.03 ); + _raveGloveParticleSystem.setEmitterThrust (_raveGloveEmitter[f], 0.0f ); + _raveGloveParticleSystem.setEmitterRate (_raveGloveEmitter[f], 30.0 ); + _raveGloveParticleSystem.setEmitterParticleResolution (_raveGloveEmitter[f], 20 ); + + _raveGloveParticleSystem.setParticleAttributesToDefault(&attributes); + + attributes.radius = 0.01f; + attributes.color = glm::vec4( 0.1f, 0.2f, 0.4f, 0.5f); + attributes.modulationAmplitude = 0.5; + attributes.modulationRate = 3.0; + attributes.modulationStyle = COLOR_MODULATION_STYLE_LIGHTNESS_WAVE; + _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_0, attributes); + _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_1, attributes); + _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_2, attributes); + _raveGloveParticleSystem.setParticleAttributes(_raveGloveEmitter[f], PARTICLE_LIFESTAGE_3, attributes); } } } diff --git a/interface/src/avatar/Hand.h b/interface/src/avatar/Hand.h index a3a00beb96..beb7b8d516 100755 --- a/interface/src/avatar/Hand.h +++ b/interface/src/avatar/Hand.h @@ -65,8 +65,9 @@ public: void setRaveGloveEffectsMode(QKeyEvent* event); // getters - const glm::vec3& getLeapBallPosition (int ball) const { return _leapBalls[ball].position;} - bool isRaveGloveActive () const { return _isRaveGloveActive; } + const glm::vec3& getLeapFingerTipBallPosition (int ball) const { return _leapFingerTipBalls [ball].position;} + const glm::vec3& getLeapFingerRootBallPosition(int ball) const { return _leapFingerRootBalls[ball].position;} + bool isRaveGloveActive() const { return _isRaveGloveActive; } private: // disallow copies of the Hand, copy of owning Avatar is disallowed too @@ -84,7 +85,8 @@ private: float _renderAlpha; bool _lookingInMirror; glm::vec3 _ballColor; - std::vector _leapBalls; + std::vector _leapFingerTipBalls; + std::vector _leapFingerRootBalls; // private methods void setLeapHands(const std::vector& handPositions, @@ -92,8 +94,10 @@ private: void renderRaveGloveStage(); void setRaveGloveMode(int mode); - void renderHandSpheres(); - void renderFingerTrails(); + void renderLeapHandSpheres(); + void renderLeapHands(); + void renderLeapHand(PalmData& hand); + void renderLeapFingerTrails(); void calculateGeometry(); }; diff --git a/libraries/avatars/CMakeLists.txt b/libraries/avatars/CMakeLists.txt index 7968b5a003..207057e244 100644 --- a/libraries/avatars/CMakeLists.txt +++ b/libraries/avatars/CMakeLists.txt @@ -8,13 +8,9 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cm set(TARGET_NAME avatars) -find_package(Qt5Core) - include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) -qt5_use_modules(${TARGET_NAME} Core) - include(${MACRO_DIR}/IncludeGLM.cmake) include_glm(${TARGET_NAME} ${ROOT_DIR}) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp old mode 100755 new mode 100644 index 65ae0c6952..9c254453e1 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -130,6 +130,8 @@ int AvatarData::getBroadcastData(unsigned char* destinationBuffer) { // leap hand data std::vector fingerVectors; + +//printf("about to call _handData->encodeRemoteData(fingerVectors);\n"); _handData->encodeRemoteData(fingerVectors); if (fingerVectors.size() > 255) @@ -244,17 +246,32 @@ int AvatarData::parseData(unsigned char* sourceBuffer, int numBytes) { // hand state, stored as a semi-nibble in the bitItems _handState = getSemiNibbleAt(bitItems,HAND_STATE_START_BIT); +//printf("about to call leap hand data code in AvatarData::parseData...\n"); + // leap hand data if (sourceBuffer - startPosition < numBytes) { + +//printf("got inside of 'if (sourceBuffer - startPosition < numBytes)'\n"); + + // check passed, bytes match unsigned int numFingerVectors = *sourceBuffer++; + +//printf("numFingerVectors = %d\n", numFingerVectors); + + if (numFingerVectors > 0) { + +//printf("ok, we got fingers in AvatarData::parseData\n"); + std::vector fingerVectors(numFingerVectors); for (size_t i = 0; i < numFingerVectors; ++i) { sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].x), fingerVectorRadix); sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].y), fingerVectorRadix); sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerVectors[i].z), fingerVectorRadix); } + +//printf("about to call _handData->decodeRemoteData(fingerVectors);\n"); _handData->decodeRemoteData(fingerVectors); } } diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp old mode 100755 new mode 100644 index 7bb6eebdc0..0b91191d92 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -51,12 +51,18 @@ _owningHandData(owningHandData) void HandData::encodeRemoteData(std::vector& fingerVectors) { fingerVectors.clear(); + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& palm = getPalms()[i]; + if (!palm.isActive()) { + continue; + } fingerVectors.push_back(palm.getRawPosition()); fingerVectors.push_back(palm.getRawNormal()); for (size_t f = 0; f < palm.getNumFingers(); ++f) { FingerData& finger = palm.getFingers()[f]; + if (finger.isActive()) { fingerVectors.push_back(finger.getTipRawPosition()); fingerVectors.push_back(finger.getRootRawPosition()); @@ -83,7 +89,8 @@ void HandData::decodeRemoteData(const std::vector& fingerVectors) { palm.setRawPosition(fingerVectors[vectorIndex++]); palm.setRawNormal(fingerVectors[vectorIndex++]); for (size_t f = 0; f < NUM_FINGERS_PER_HAND; ++f) { - FingerData& finger = palm.getFingers()[i]; + FingerData& finger = palm.getFingers()[f]; + finger.setActive(true); finger.setRawTipPosition(fingerVectors[vectorIndex++]); finger.setRawRootPosition(fingerVectors[vectorIndex++]); } diff --git a/libraries/shared/CMakeLists.txt b/libraries/shared/CMakeLists.txt index b0c2771f48..cf1c603b7d 100644 --- a/libraries/shared/CMakeLists.txt +++ b/libraries/shared/CMakeLists.txt @@ -6,13 +6,9 @@ set(MACRO_DIR ${ROOT_DIR}/cmake/macros) set(TARGET_NAME shared) project(${TARGET_NAME}) -find_package(Qt5Core REQUIRED) - include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) -qt5_use_modules(${TARGET_NAME} Core) - set(EXTERNAL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external) if (WIN32) diff --git a/libraries/voxels/CMakeLists.txt b/libraries/voxels/CMakeLists.txt index 861001ed35..b6f7ae9718 100644 --- a/libraries/voxels/CMakeLists.txt +++ b/libraries/voxels/CMakeLists.txt @@ -8,9 +8,13 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cm set(TARGET_NAME voxels) +find_package(Qt5Widgets REQUIRED) + include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) +qt5_use_modules(${TARGET_NAME} Widgets) + include(${MACRO_DIR}/IncludeGLM.cmake) include_glm(${TARGET_NAME} ${ROOT_DIR}) diff --git a/libraries/voxels/src/VoxelConstants.h b/libraries/voxels/src/VoxelConstants.h index 55dff0c69a..4b971e9515 100644 --- a/libraries/voxels/src/VoxelConstants.h +++ b/libraries/voxels/src/VoxelConstants.h @@ -41,4 +41,6 @@ const glBufferIndex GLBUFFER_INDEX_UNKNOWN = ULONG_MAX; const float SIXTY_FPS_IN_MILLISECONDS = 1000.0f / 60.0f; const float VIEW_CULLING_RATE_IN_MILLISECONDS = 1000.0f; // once a second is fine +const uint64_t CLIENT_TO_SERVER_VOXEL_SEND_INTERVAL_USECS = 1000 * 5; // 1 packet every 50 milliseconds + #endif \ No newline at end of file diff --git a/voxel-server/src/main.cpp b/voxel-server/src/main.cpp index 0854dfa4b7..216c576b2f 100644 --- a/voxel-server/src/main.cpp +++ b/voxel-server/src/main.cpp @@ -61,6 +61,7 @@ bool wantColorRandomizer = false; bool debugVoxelSending = false; bool shouldShowAnimationDebug = false; bool displayVoxelStats = false; +bool debugVoxelReceiving = false; EnvironmentData environmentData[3]; @@ -426,6 +427,8 @@ void attachVoxelNodeDataToNode(Node* newNode) { } } +int receivedPacketCount = 0; + int main(int argc, const char * argv[]) { pthread_mutex_init(&::treeLock, NULL); @@ -455,6 +458,10 @@ int main(int argc, const char * argv[]) { ::debugVoxelSending = cmdOptionExists(argc, argv, DEBUG_VOXEL_SENDING); printf("debugVoxelSending=%s\n", debug::valueOf(::debugVoxelSending)); + const char* DEBUG_VOXEL_RECEIVING = "--debugVoxelReceiving"; + ::debugVoxelReceiving = cmdOptionExists(argc, argv, DEBUG_VOXEL_RECEIVING); + printf("debugVoxelReceiving=%s\n", debug::valueOf(::debugVoxelReceiving)); + const char* WANT_ANIMATION_DEBUG = "--shouldShowAnimationDebug"; ::shouldShowAnimationDebug = cmdOptionExists(argc, argv, WANT_ANIMATION_DEBUG); printf("shouldShowAnimationDebug=%s\n", debug::valueOf(::shouldShowAnimationDebug)); @@ -584,12 +591,20 @@ int main(int argc, const char * argv[]) { destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", ::shouldShowAnimationDebug); + ::receivedPacketCount++; + unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader))); if (::shouldShowAnimationDebug) { printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n", destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", receivedBytes,itemNumber); } + + if (::debugVoxelReceiving) { + printf("got %s - %d command from client receivedBytes=%ld itemNumber=%d\n", + destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL", + ::receivedPacketCount, receivedBytes,itemNumber); + } int atByte = numBytesPacketHeader + sizeof(itemNumber); unsigned char* voxelData = (unsigned char*)&packetData[atByte]; while (atByte < receivedBytes) {