Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-16 10:46:45 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of https://github.com/highfidelity/hifi into metavoxels

Browse source
This commit is contained in:
Andrzej Kapolka 2014-10-10 17:19:24 -07:00
commit dc8dc112c9
11 changed files with 222 additions and 661 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
interface/src/Application.cpp
View file

@ -1089,6 +1089,9 @@ void Application::keyPressEvent(QKeyEvent* event) {
case Qt::Key_Equal:
_myAvatar->resetSize();
break;
case Qt::Key_Escape:
OculusManager::abandonCalibration();
break;
default:
event->ignore();
break;
@ -1481,6 +1484,9 @@ void Application::setEnableVRMode(bool enableVRMode) {
OculusManager::disconnect();
OculusManager::connect();
}
OculusManager::recalibrate();
} else {
OculusManager::abandonCalibration();
}
resizeGL(_glWidget->getDeviceWidth(), _glWidget->getDeviceHeight());
@ -1545,10 +1551,9 @@ glm::vec3 Application::getMouseVoxelWorldCoordinates(const VoxelDetail& mouseVox
FaceTracker* Application::getActiveFaceTracker() {
return (_dde.isActive() ? static_cast<FaceTracker*>(&_dde) :
(_cara.isActive() ? static_cast<FaceTracker*>(&_cara) :
(_faceshift.isActive() ? static_cast<FaceTracker*>(&_faceshift) :
(_faceplus.isActive() ? static_cast<FaceTracker*>(&_faceplus) :
(_visage.isActive() ? static_cast<FaceTracker*>(&_visage) : NULL)))));
(_visage.isActive() ? static_cast<FaceTracker*>(&_visage) : NULL))));
}
struct SendVoxelsOperationArgs {
@ -2007,19 +2012,6 @@ void Application::updateDDE() {
_dde.update();
}
void Application::updateCara() {
bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings);
PerformanceWarning warn(showWarnings, "Application::updateCara()");
// Update Cara
_cara.update();
// Copy angular velocity if measured by cara, to the head
if (_cara.isActive()) {
_myAvatar->getHead()->setAngularVelocity(_cara.getHeadAngularVelocity());
}
}
void Application::updateMyAvatarLookAtPosition() {
PerformanceTimer perfTimer("lookAt");
bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings);
@ -2119,7 +2111,6 @@ void Application::updateMetavoxels(float deltaTime) {
}
void Application::cameraMenuChanged() {
float modeShiftPeriod = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? 0.0f : 1.0f;
if (Menu::getInstance()->isOptionChecked(MenuOption::FullscreenMirror)) {
if (_myCamera.getMode() != CAMERA_MODE_MIRROR) {
_myCamera.setMode(CAMERA_MODE_MIRROR);

7
interface/src/Application.h
View file

@ -63,7 +63,6 @@
#include "devices/PrioVR.h"
#include "devices/SixenseManager.h"
#include "devices/Visage.h"
#include "devices/CaraFaceTracker.h"
#include "devices/DdeFaceTracker.h"
#include "entities/EntityTreeRenderer.h"
#include "particles/ParticleTreeRenderer.h"
@ -219,7 +218,6 @@ public:
Faceshift* getFaceshift() { return &_faceshift; }
Visage* getVisage() { return &_visage; }
DdeFaceTracker* getDDE() { return &_dde; }
CaraFaceTracker* getCara() { return &_cara; }
FaceTracker* getActiveFaceTracker();
PrioVR* getPrioVR() { return &_prioVR; }
BandwidthMeter* getBandwidthMeter() { return &_bandwidthMeter; }
@ -365,6 +363,8 @@ public slots:
void domainSettingsReceived(const QJsonObject& domainSettingsObject);
void resetSensors();
private slots:
void timer();
void idle();
@ -381,7 +381,6 @@ private slots:
void closeMirrorView();
void restoreMirrorView();
void shrinkMirrorView();
void resetSensors();
void parseVersionXml();
@ -407,7 +406,6 @@ private:
void updateFaceshift();
void updateVisage();
void updateDDE();
void updateCara();
void updateMyAvatarLookAtPosition();
void updateThreads(float deltaTime);
void updateMetavoxels(float deltaTime);
@ -508,7 +506,6 @@ private:
Faceplus _faceplus;
Faceshift _faceshift;
Visage _visage;
CaraFaceTracker _cara;
DdeFaceTracker _dde;
PrioVR _prioVR;

455
interface/src/devices/CaraFaceTracker.cpp
View file

@ -1,455 +0,0 @@
//
// CaraFaceTracker.cpp
// interface/src/devices
//
// Created by Li Zuwei on 7/22/14.
// Copyright 2014 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "CaraFaceTracker.h"
#include <GLMHelpers.h>
//qt
#include <QJsonDocument>
#include <QJsonArray>
#include <QJsonObject>
#include <QElapsedTimer>
#define PI M_PI
#define RADTODEG(x) ( (x) * 180.0 / PI )
#define DEGTORAD(x) ( (x) * PI / 180.0 )
static const QHostAddress CARA_FEATURE_POINT_SERVER_ADDR("127.0.0.1");
static const quint16 CARA_FEATURE_POINT_SERVER_PORT = 36555;
static QString sampleJson = "[{\"id\":1, \
\"face\":{\"x\":248,\"y\":64,\"width\":278,\"height\":341}, \
\"pose\":{\"roll\":2.62934,\"pitch\":-12.2318,\"yaw\":0.936743}, \
\"feature_points\":[314,194,326,187,340,187,354,189,367,193,409,190,421,187,435,184,448,183,459,188, \
388,207,389,223,390,240,391,257,377,266,384,267,392,268,399,266,407,264,331,209, \
341,204,354,204,364,209,353,214,341,214,410,208,420,201,433,200,443,205,434,211, \
421,211,362,294,372,290,383,287,393,289,404,286,415,289,426,291,418,300,407,306, \
394,308,382,307,371,302,383,295,394,295,404,294,404,295,393,297,383,296], \
\"classifiers\":{\"emotion\":{\"smi\":-0.368829,\"sur\":-1.33334,\"neg\":0.00235828,\"att\":1},\"blink\":1}}]";
static const glm::vec3 DEFAULT_HEAD_ORIGIN(0.0f, 0.0f, 0.0f);
static const float TRANSLATION_SCALE = 1.0f;
static const int NUM_BLENDSHAPE_COEFF = 30;
static const int NUM_SMOOTHING_SAMPLES = 3;
struct CaraPerson {
struct CaraPose {
float roll, pitch, yaw;
CaraPose() :
roll(0.0f),
pitch(0.0f),
yaw(0.0f)
{
}
};
struct CaraEmotion {
float smile, surprise, negative, attention;
CaraEmotion():
smile(0.0f),
surprise(0.0f),
negative(0.0f),
attention(0.0f)
{
}
};
enum CaraBlink {
BLINK_NOT_AVAILABLE,
NO_BLINK,
BLINK
};
CaraPerson() :
id(-1),
blink(BLINK_NOT_AVAILABLE)
{
}
int id;
CaraPose pose;
CaraEmotion emotion;
CaraBlink blink;
QString toString() {
QString s = QString("id: %1, roll: %2, pitch: %3, yaw: %4, smi: %5, sur: %6, neg: %7, att: %8, blink: %9").
arg(id).
arg(pose.roll).
arg(pose.pitch).
arg(pose.yaw).
arg(emotion.smile).
arg(emotion.surprise).
arg(emotion.negative).
arg(emotion.attention).
arg(blink);
return s;
}
};
class CaraPacketDecoder {
public:
static CaraPerson extractOne(const QByteArray& buffer, QJsonParseError* jsonError) {
CaraPerson person;
QJsonDocument dom = QJsonDocument::fromJson(buffer, jsonError);
//check for errors
if(jsonError->error == QJsonParseError::NoError) {
//read the dom structure and populate the blend shapes and head poses
//qDebug() << "[Info] Cara Face Tracker Packet Parsing Successful!";
//begin extracting the packet
if(dom.isArray()) {
QJsonArray people = dom.array();
//extract the first person in the array
if(people.size() > 0) {
QJsonValue val = people.at(0);
if(val.isObject()) {
QJsonObject personDOM = val.toObject();
person.id = extractId(personDOM);
person.pose = extractPose(personDOM);
//extract the classifier outputs
QJsonObject::const_iterator it = personDOM.constFind("classifiers");
if(it != personDOM.constEnd()) {
QJsonObject classifierDOM = (*it).toObject();
person.emotion = extractEmotion(classifierDOM);
person.blink = extractBlink(classifierDOM);
}
}
}
}
}
return person;
}
private:
static int extractId(const QJsonObject& person) {
int id = -1;
QJsonObject::const_iterator it = person.constFind("id");
if(it != person.constEnd()) {
id = (*it).toInt(-1);
}
return id;
}
static CaraPerson::CaraPose extractPose(const QJsonObject& person) {
CaraPerson::CaraPose pose;
QJsonObject::const_iterator it = person.constFind("pose");
if(it != person.constEnd()) {
QJsonObject poseDOM = (*it).toObject();
//look for the roll, pitch, yaw;
QJsonObject::const_iterator poseIt = poseDOM.constFind("roll");
QJsonObject::const_iterator poseEnd = poseDOM.constEnd();
if(poseIt != poseEnd) {
pose.roll = (float)(*poseIt).toDouble(0.0);
}
poseIt = poseDOM.constFind("pitch");
if(poseIt != poseEnd) {
pose.pitch = (float)(*poseIt).toDouble(0.0);
}
poseIt = poseDOM.constFind("yaw");
if(poseIt != poseEnd) {
pose.yaw = (float)(*poseIt).toDouble(0.0);
}
}
return pose;
}
static CaraPerson::CaraEmotion extractEmotion(const QJsonObject& classifiers) {
CaraPerson::CaraEmotion emotion;
QJsonObject::const_iterator it = classifiers.constFind("emotion");
if(it != classifiers.constEnd()) {
QJsonObject emotionDOM = (*it).toObject();
//look for smile, surprise, negative, attention responses
QJsonObject::const_iterator emoEnd = emotionDOM.constEnd();
QJsonObject::const_iterator emoIt = emotionDOM.constFind("smi");
if(emoIt != emoEnd) {
emotion.smile = (float)(*emoIt).toDouble(0.0);
}
emoIt = emotionDOM.constFind("sur");
if(emoIt != emoEnd) {
emotion.surprise = (float)(*emoIt).toDouble(0.0);
}
emoIt = emotionDOM.constFind("neg");
if(emoIt != emoEnd) {
emotion.negative = (float)(*emoIt).toDouble(0.0);
}
emoIt = emotionDOM.constFind("att");
if(emoIt != emoEnd) {
emotion.attention = (float)(*emoIt).toDouble(0.0);
}
}
return emotion;
}
static CaraPerson::CaraBlink extractBlink(const QJsonObject& classifiers) {
CaraPerson::CaraBlink blink = CaraPerson::BLINK_NOT_AVAILABLE;
QJsonObject::const_iterator it = classifiers.constFind("blink");
if(it != classifiers.constEnd()) {
int b = (*it).toInt(CaraPerson::BLINK_NOT_AVAILABLE);
switch(b) {
case CaraPerson::BLINK_NOT_AVAILABLE:
blink = CaraPerson::BLINK_NOT_AVAILABLE;
break;
case CaraPerson::NO_BLINK:
blink = CaraPerson::NO_BLINK;
break;
case CaraPerson::BLINK:
blink = CaraPerson::BLINK;
break;
default:
blink = CaraPerson::BLINK_NOT_AVAILABLE;
break;
}
}
return blink;
}
};
CaraFaceTracker::CaraFaceTracker() :
_lastReceiveTimestamp(0),
_pitchAverage(NUM_SMOOTHING_SAMPLES),
_yawAverage(NUM_SMOOTHING_SAMPLES),
_rollAverage(NUM_SMOOTHING_SAMPLES),
_eyeGazeLeftPitch(0.0f),
_eyeGazeLeftYaw(0.0f),
_eyeGazeRightPitch(0.0f),
_eyeGazeRightYaw(0),
_leftBlinkIndex(0),
_rightBlinkIndex(1),
_leftEyeOpenIndex(8),
_rightEyeOpenIndex(9),
_browDownLeftIndex(14),
_browDownRightIndex(15),
_browUpCenterIndex(16),
_browUpLeftIndex(17),
_browUpRightIndex(18),
_mouthSmileLeftIndex(28),
_mouthSmileRightIndex(29),
_jawOpenIndex(21)
{
connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams()));
connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError)));
connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), SLOT(socketStateChanged(QAbstractSocket::SocketState)));
bindTo(CARA_FEATURE_POINT_SERVER_PORT);
_headTranslation = DEFAULT_HEAD_ORIGIN;
_blendshapeCoefficients.resize(NUM_BLENDSHAPE_COEFF);
_blendshapeCoefficients.fill(0.0f);
//qDebug() << sampleJson;
}
CaraFaceTracker::CaraFaceTracker(const QHostAddress& host, quint16 port) :
_lastReceiveTimestamp(0),
_pitchAverage(NUM_SMOOTHING_SAMPLES),
_yawAverage(NUM_SMOOTHING_SAMPLES),
_rollAverage(NUM_SMOOTHING_SAMPLES),
_eyeGazeLeftPitch(0.0f),
_eyeGazeLeftYaw(0.0f),
_eyeGazeRightPitch(0.0f),
_eyeGazeRightYaw(0.0f),
_leftBlinkIndex(0),
_rightBlinkIndex(1),
_leftEyeOpenIndex(8),
_rightEyeOpenIndex(9),
_browDownLeftIndex(14),
_browDownRightIndex(15),
_browUpCenterIndex(16),
_browUpLeftIndex(17),
_browUpRightIndex(18),
_mouthSmileLeftIndex(28),
_mouthSmileRightIndex(29),
_jawOpenIndex(21)
{
connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams()));
connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError)));
connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), SIGNAL(socketStateChanged(QAbstractSocket::SocketState)));
bindTo(host, port);
_headTranslation = DEFAULT_HEAD_ORIGIN * TRANSLATION_SCALE;
_blendshapeCoefficients.resize(NUM_BLENDSHAPE_COEFF); //set the size of the blendshape coefficients
_blendshapeCoefficients.fill(0.0f);
}
CaraFaceTracker::~CaraFaceTracker() {
if(_udpSocket.isOpen())
_udpSocket.close();
}
void CaraFaceTracker::init() {
}
void CaraFaceTracker::reset() {
}
void CaraFaceTracker::bindTo(quint16 port) {
bindTo(QHostAddress::Any, port);
}
void CaraFaceTracker::bindTo(const QHostAddress& host, quint16 port) {
if(_udpSocket.isOpen()) {
_udpSocket.close();
}
_udpSocket.bind(host, port);
}
bool CaraFaceTracker::isActive() const {
static const quint64 ACTIVE_TIMEOUT_USECS = 3000000; //3 secs
return (usecTimestampNow() - _lastReceiveTimestamp < ACTIVE_TIMEOUT_USECS);
}
void CaraFaceTracker::update() {
// get the euler angles relative to the window
glm::vec3 eulers = glm::degrees(safeEulerAngles(_headRotation * glm::quat(glm::radians(glm::vec3(
(_eyeGazeLeftPitch + _eyeGazeRightPitch) / 2.0f, (_eyeGazeLeftYaw + _eyeGazeRightYaw) / 2.0f, 0.0f)))));
//TODO: integrate when cara has eye gaze estimation
_estimatedEyePitch = eulers.x;
_estimatedEyeYaw = eulers.y;
}
//private slots and methods
void CaraFaceTracker::socketErrorOccurred(QAbstractSocket::SocketError socketError) {
qDebug() << "[Error] Cara Face Tracker Socket Error: " << _udpSocket.errorString();
}
void CaraFaceTracker::socketStateChanged(QAbstractSocket::SocketState socketState) {
QString state;
switch(socketState) {
case QAbstractSocket::BoundState:
state = "Bounded";
break;
case QAbstractSocket::ClosingState:
state = "Closing";
break;
case QAbstractSocket::ConnectedState:
state = "Connected";
break;
case QAbstractSocket::ConnectingState:
state = "Connecting";
break;
case QAbstractSocket::HostLookupState:
state = "Host Lookup";
break;
case QAbstractSocket::ListeningState:
state = "Listening";
break;
case QAbstractSocket::UnconnectedState:
state = "Unconnected";
break;
}
qDebug() << "[Info] Cara Face Tracker Socket: " << socketState;
}
void CaraFaceTracker::readPendingDatagrams() {
QByteArray buffer;
while (_udpSocket.hasPendingDatagrams()) {
buffer.resize(_udpSocket.pendingDatagramSize());
_udpSocket.readDatagram(buffer.data(), buffer.size());
decodePacket(buffer);
}
}
void CaraFaceTracker::decodePacket(const QByteArray& buffer) {
//decode the incoming udp packet
QJsonParseError jsonError;
CaraPerson person = CaraPacketDecoder::extractOne(buffer, &jsonError);
if(jsonError.error == QJsonParseError::NoError) {
//do some noise filtering to the head poses
//reduce the noise first by truncating to 1 dp
person.pose.roll = glm::floor(person.pose.roll * 10) / 10;
person.pose.pitch = glm::floor(person.pose.pitch * 10) / 10;
person.pose.yaw = glm::floor(person.pose.yaw * 10) / 10;
//qDebug() << person.toString();
glm::quat newRotation(glm::vec3(DEGTORAD(person.pose.pitch), DEGTORAD(person.pose.yaw), DEGTORAD(person.pose.roll)));
// Compute angular velocity of the head
glm::quat r = newRotation * glm::inverse(_headRotation);
float theta = 2 * acos(r.w);
if (theta > EPSILON) {
float rMag = glm::length(glm::vec3(r.x, r.y, r.z));
const float AVERAGE_CARA_FRAME_TIME = 0.04f;
const float YAW_STANDARD_DEV_DEG = 2.5f;
_headAngularVelocity = theta / AVERAGE_CARA_FRAME_TIME * glm::vec3(r.x, r.y, r.z) / rMag;
_pitchAverage.updateAverage(person.pose.pitch);
_rollAverage.updateAverage(person.pose.roll);
//could use the angular velocity to detemine whether to update pitch and roll to further remove the noise.
//use the angular velocity for roll and pitch, update if > THRESHOLD
//if(glm::abs(_headAngularVelocity.x) > ANGULAR_VELOCITY_MIN) {
// _pitchAverage.updateAverage(person.pose.pitch);
//}
//if(glm::abs(_headAngularVelocity.z) > ANGULAR_VELOCITY_MIN) {
// _rollAverage.updateAverage(person.pose.roll);;
//}
//for yaw, the jitter is great, you can't use angular velocity because it swings too much
//use the previous and current yaw, calculate the
//abs difference and move it the difference is above the standard deviation which is around 2.5
// > the standard deviation 2.5 deg, update the yaw smoothing average
if(glm::abs(person.pose.yaw - _yawAverage.getAverage()) > YAW_STANDARD_DEV_DEG) {
//qDebug() << "Yaw Diff: " << glm::abs(person.pose.yaw - _previousYaw);
_yawAverage.updateAverage(person.pose.yaw);
}
//set the new rotation
newRotation = glm::quat(glm::vec3(DEGTORAD(_pitchAverage.getAverage()), DEGTORAD(_yawAverage.getAverage()), DEGTORAD(-_rollAverage.getAverage())));
}
else {
//no change in position, use previous averages
newRotation = glm::quat(glm::vec3(DEGTORAD(_pitchAverage.getAverage()), DEGTORAD(_yawAverage.getAverage()), DEGTORAD(-_rollAverage.getAverage())));
_headAngularVelocity = glm::vec3(0,0,0);
}
//update to new rotation angles
_headRotation = newRotation;
//TODO: head translation, right now is 0
//Do Blendshapes, clip between 0.0f to 1.0f, neg should be ignored
_blendshapeCoefficients[_leftBlinkIndex] = person.blink == CaraPerson::BLINK ? 1.0f : 0.0f;
_blendshapeCoefficients[_rightBlinkIndex] = person.blink == CaraPerson::BLINK ? 1.0f : 0.0f;
//anger and surprised are mutually exclusive so we could try use this fact to determine
//whether to down the brows or up the brows
_blendshapeCoefficients[_browDownLeftIndex] = person.emotion.negative < 0.0f ? 0.0f : person.emotion.negative;
_blendshapeCoefficients[_browDownRightIndex] = person.emotion.negative < 0.0f ? 0.0f : person.emotion.negative;
_blendshapeCoefficients[_browUpCenterIndex] = person.emotion.surprise < 0.0f ? 0.0f : person.emotion.surprise;
_blendshapeCoefficients[_browUpLeftIndex] = person.emotion.surprise < 0.0f ? 0.0f : person.emotion.surprise;
_blendshapeCoefficients[_browUpRightIndex] = person.emotion.surprise < 0.0f ? 0.0f : person.emotion.surprise;
_blendshapeCoefficients[_jawOpenIndex] = person.emotion.surprise < 0.0f ? 0.0f : person.emotion.surprise;
_blendshapeCoefficients[_mouthSmileLeftIndex] = person.emotion.smile < 0.0f ? 0.0f : person.emotion.smile;
_blendshapeCoefficients[_mouthSmileRightIndex] = person.emotion.smile < 0.0f ? 0.0f : person.emotion.smile;
}
else {
qDebug() << "[Error] Cara Face Tracker Decode Error: " << jsonError.errorString();
}
_lastReceiveTimestamp = usecTimestampNow();
}
float CaraFaceTracker::getBlendshapeCoefficient(int index) const {
return (index >= 0 && index < (int)_blendshapeCoefficients.size()) ? _blendshapeCoefficients[index] : 0.0f;
}

124
interface/src/devices/CaraFaceTracker.h
View file

@ -1,124 +0,0 @@
//
// CaraFaceTracker.h
// interface/src/devices
//
// Created by Li Zuwei on 7/22/14.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_CaraFaceTracker_h
#define hifi_CaraFaceTracker_h
#include <QUdpSocket>
#include <SimpleMovingAverage.h>
#include "FaceTracker.h"
/*!
* \class CaraFaceTracker
*
* \brief Handles interaction with the Cara software,
* which provides head position/orientation and facial features.
* \details By default, opens a udp socket with IPV4_ANY_ADDR with port 36555.
* User needs to run the Cara Face Detection UDP Client with the destination
* host address (eg: 127.0.0.1 for localhost) and destination port 36555.
**/
class CaraFaceTracker : public FaceTracker {
Q_OBJECT
public:
CaraFaceTracker();
CaraFaceTracker(const QHostAddress& host, quint16 port);
~CaraFaceTracker();
//initialization
void init();
void reset();
//sockets
void bindTo(quint16 port);
void bindTo(const QHostAddress& host, quint16 port);
bool isActive() const;
//tracking
void update();
//head angular velocity
const glm::vec3& getHeadAngularVelocity() const { return _headAngularVelocity; }
//eye gaze
float getEyeGazeLeftPitch() const { return _eyeGazeLeftPitch; }
float getEyeGazeLeftYaw() const { return _eyeGazeLeftYaw; }
float getEyeGazeRightPitch() const { return _eyeGazeRightPitch; }
float getEyeGazeRightYaw() const { return _eyeGazeRightYaw; }
//blend shapes
float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); }
float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); }
float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); }
float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); }
float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); }
float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); }
float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); }
float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); }
float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); }
float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); }
float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); }
float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); }
private slots:
//sockets
void socketErrorOccurred(QAbstractSocket::SocketError socketError);
void readPendingDatagrams();
void socketStateChanged(QAbstractSocket::SocketState socketState);
private:
void decodePacket(const QByteArray& buffer);
float getBlendshapeCoefficient(int index) const;
// sockets
QUdpSocket _udpSocket;
quint64 _lastReceiveTimestamp;
//head tracking
glm::vec3 _headAngularVelocity;
//pose average
SimpleMovingAverage _pitchAverage;
SimpleMovingAverage _yawAverage;
SimpleMovingAverage _rollAverage;
// eye gaze degrees
float _eyeGazeLeftPitch;
float _eyeGazeLeftYaw;
float _eyeGazeRightPitch;
float _eyeGazeRightYaw;
//blend shapes
int _leftBlinkIndex;
int _rightBlinkIndex;
int _leftEyeOpenIndex;
int _rightEyeOpenIndex;
// Brows
int _browDownLeftIndex;
int _browDownRightIndex;
int _browUpCenterIndex;
int _browUpLeftIndex;
int _browUpRightIndex;
int _mouthSmileLeftIndex;
int _mouthSmileRightIndex;
int _jawOpenIndex;
};
#endif //endif hifi_CaraFaceTracker_h

141
interface/src/devices/OculusManager.cpp
View file

@ -55,6 +55,20 @@ bool OculusManager::_programInitialized = false;
Camera* OculusManager::_camera = NULL;
int OculusManager::_activeEyeIndex = -1;
float OculusManager::CALIBRATION_DELTA_MINIMUM_LENGTH = 0.02f;
float OculusManager::CALIBRATION_DELTA_MINIMUM_ANGLE = 5.f * RADIANS_PER_DEGREE;
float OculusManager::CALIBRATION_ZERO_MAXIMUM_LENGTH = 0.01f;
float OculusManager::CALIBRATION_ZERO_MAXIMUM_ANGLE = 2.0f * RADIANS_PER_DEGREE;
quint64 OculusManager::CALIBRATION_ZERO_HOLD_TIME = 3000000; // usec
float OculusManager::CALIBRATION_MESSAGE_DISTANCE = 2.5f;
OculusManager::CalibrationState OculusManager::_calibrationState;
glm::vec3 OculusManager::_calibrationPosition;
glm::quat OculusManager::_calibrationOrientation;
quint64 OculusManager::_calibrationStartTime;
int OculusManager::_calibrationMessage = NULL;
QString OculusManager::CALIBRATION_BILLBOARD_URL = "http://hifi-public.s3.amazonaws.com/images/hold-to-calibrate.svg";
float OculusManager::CALIBRATION_BILLBOARD_SCALE = 2.f;
#endif
glm::vec3 OculusManager::_leftEyePosition = glm::vec3();
@ -62,6 +76,8 @@ glm::vec3 OculusManager::_rightEyePosition = glm::vec3();
void OculusManager::connect() {
#ifdef HAVE_LIBOVR
_calibrationState = UNCALIBRATED;
ovr_Initialize();
_ovrHmd = ovrHmd_Create(0);
@ -172,6 +188,124 @@ void OculusManager::disconnect() {
#endif
}
#ifdef HAVE_LIBOVR
void OculusManager::positionCalibrationBillboard(BillboardOverlay* billboard) {
glm::quat headOrientation = Application::getInstance()->getAvatar()->getHeadOrientation();
headOrientation.x = 0;
headOrientation.z = 0;
glm::normalize(headOrientation);
billboard->setPosition(Application::getInstance()->getAvatar()->getHeadPosition()
+ headOrientation * glm::vec3(0.f, 0.f, -CALIBRATION_MESSAGE_DISTANCE));
billboard->setRotation(headOrientation);
}
#endif
#ifdef HAVE_LIBOVR
void OculusManager::calibrate(glm::vec3 position, glm::quat orientation) {
static QString progressMessage;
static BillboardOverlay* billboard;
switch (_calibrationState) {
case UNCALIBRATED:
if (position != glm::vec3() && orientation != glm::quat()) { // Handle zero values at start-up.
_calibrationPosition = position;
_calibrationOrientation = orientation;
_calibrationState = WAITING_FOR_DELTA;
}
break;
case WAITING_FOR_DELTA:
if (glm::length(position - _calibrationPosition) > CALIBRATION_DELTA_MINIMUM_LENGTH
|| glm::angle(orientation * glm::inverse(_calibrationOrientation)) > CALIBRATION_DELTA_MINIMUM_ANGLE) {
_calibrationPosition = position;
_calibrationOrientation = orientation;
_calibrationState = WAITING_FOR_ZERO;
}
break;
case WAITING_FOR_ZERO:
if (glm::length(position - _calibrationPosition) < CALIBRATION_ZERO_MAXIMUM_LENGTH
&& glm::angle(orientation * glm::inverse(_calibrationOrientation)) < CALIBRATION_ZERO_MAXIMUM_ANGLE) {
_calibrationStartTime = usecTimestampNow();
_calibrationState = WAITING_FOR_ZERO_HELD;
if (!_calibrationMessage) {
qDebug() << "Hold still to calibrate HMD";
billboard = new BillboardOverlay();
billboard->setURL(CALIBRATION_BILLBOARD_URL);
billboard->setScale(CALIBRATION_BILLBOARD_SCALE);
billboard->setIsFacingAvatar(false);
positionCalibrationBillboard(billboard);
_calibrationMessage = Application::getInstance()->getOverlays().addOverlay(billboard);
}
progressMessage = "";
} else {
_calibrationPosition = position;
_calibrationOrientation = orientation;
}
break;
case WAITING_FOR_ZERO_HELD:
if (glm::length(position - _calibrationPosition) < CALIBRATION_ZERO_MAXIMUM_LENGTH
&& glm::angle(orientation * glm::inverse(_calibrationOrientation)) < CALIBRATION_ZERO_MAXIMUM_ANGLE) {
if ((usecTimestampNow() - _calibrationStartTime) > CALIBRATION_ZERO_HOLD_TIME) {
_calibrationState = CALIBRATED;
qDebug() << "HMD calibrated";
Application::getInstance()->getOverlays().deleteOverlay(_calibrationMessage);
_calibrationMessage = NULL;
Application::getInstance()->resetSensors();
} else {
quint64 quarterSeconds = (usecTimestampNow() - _calibrationStartTime) / 250000;
if (quarterSeconds + 1 > progressMessage.length()) {
// 3...2...1...
if (quarterSeconds == 4 * (quarterSeconds / 4)) {
quint64 wholeSeconds = CALIBRATION_ZERO_HOLD_TIME / 1000000 - quarterSeconds / 4;
if (wholeSeconds == 3) {
positionCalibrationBillboard(billboard);
}
progressMessage += QString::number(wholeSeconds);
} else {
progressMessage += ".";
}
//qDebug() << progressMessage; // Progress message ready for 3D text overlays.
}
}
} else {
_calibrationPosition = position;
_calibrationOrientation = orientation;
_calibrationState = WAITING_FOR_ZERO;
}
break;
default:
break;
}
}
#endif
void OculusManager::recalibrate() {
#ifdef HAVE_LIBOVR
_calibrationState = UNCALIBRATED;
#endif
}
void OculusManager::abandonCalibration() {
#ifdef HAVE_LIBOVR
_calibrationState = CALIBRATED;
if (_calibrationMessage) {
qDebug() << "Abandoned HMD calibration";
Application::getInstance()->getOverlays().deleteOverlay(_calibrationMessage);
_calibrationMessage = NULL;
}
#endif
}
#ifdef HAVE_LIBOVR
void OculusManager::generateDistortionMesh() {
@ -325,6 +459,13 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p
ovrVector3f ovrHeadPosition = ts.HeadPose.ThePose.Position;
trackerPosition = glm::vec3(ovrHeadPosition.x, ovrHeadPosition.y, ovrHeadPosition.z);
if (_calibrationState != CALIBRATED) {
ovrQuatf ovrHeadOrientation = ts.HeadPose.ThePose.Orientation;
orientation = glm::quat(ovrHeadOrientation.w, ovrHeadOrientation.x, ovrHeadOrientation.y, ovrHeadOrientation.z);
calibrate(trackerPosition, orientation);
}
trackerPosition = bodyOrientation * trackerPosition;
#endif

27
interface/src/devices/OculusManager.h
View file

@ -18,6 +18,7 @@
#endif
#include "renderer/ProgramObject.h"
#include "ui/overlays/BillboardOverlay.h"
const float DEFAULT_OCULUS_UI_ANGULAR_SIZE = 72.0f;
@ -30,6 +31,8 @@ public:
static void connect();
static void disconnect();
static bool isConnected();
static void recalibrate();
static void abandonCalibration();
static void beginFrameTiming();
static void endFrameTiming();
static void configureCamera(Camera& camera, int screenWidth, int screenHeight);
@ -99,6 +102,30 @@ private:
static bool _programInitialized;
static Camera* _camera;
static int _activeEyeIndex;
static void calibrate(const glm::vec3 position, const glm::quat orientation);
enum CalibrationState {
UNCALIBRATED,
WAITING_FOR_DELTA,
WAITING_FOR_ZERO,
WAITING_FOR_ZERO_HELD,
CALIBRATED
};
static void positionCalibrationBillboard(BillboardOverlay* billboard);
static float CALIBRATION_DELTA_MINIMUM_LENGTH;
static float CALIBRATION_DELTA_MINIMUM_ANGLE;
static float CALIBRATION_ZERO_MAXIMUM_LENGTH;
static float CALIBRATION_ZERO_MAXIMUM_ANGLE;
static quint64 CALIBRATION_ZERO_HOLD_TIME;
static float CALIBRATION_MESSAGE_DISTANCE;
static CalibrationState _calibrationState;
static glm::vec3 _calibrationPosition;
static glm::quat _calibrationOrientation;
static quint64 _calibrationStartTime;
static int _calibrationMessage;
static QString CALIBRATION_BILLBOARD_URL;
static float CALIBRATION_BILLBOARD_SCALE;
#endif
static glm::vec3 _leftEyePosition;

8
interface/src/gpu/Resource.cpp
View file

@ -44,16 +44,16 @@ void Resource::Sysmem::deallocateMemory(Byte* dataAllocated, Size size) {
}
Resource::Sysmem::Sysmem() :
_data(NULL),
_stamp(0),
_size(0),
_stamp(0)
_data(NULL)
{
}
Resource::Sysmem::Sysmem(Size size, const Byte* bytes) :
_data(NULL),
_stamp(0),
_size(0),
_stamp(0)
_data(NULL)
{
if (size > 0) {
_size = allocateMemory(&_data, size);

16
interface/src/ui/overlays/BillboardOverlay.cpp
View file

@ -15,11 +15,12 @@
#include "BillboardOverlay.h"
BillboardOverlay::BillboardOverlay()
: _fromImage(-1,-1,-1,-1),
_scale(1.0f),
_isFacingAvatar(true),
_newTextureNeeded(true) {
BillboardOverlay::BillboardOverlay() :
_newTextureNeeded(true),
_fromImage(-1,-1,-1,-1),
_scale(1.0f),
_isFacingAvatar(true)
{
_isLoaded = false;
}
@ -156,9 +157,14 @@ void BillboardOverlay::setProperties(const QScriptValue &properties) {
}
}
void BillboardOverlay::setURL(const QString& url) {
setBillboardURL(url);
}
void BillboardOverlay::setBillboardURL(const QString& url) {
_url = url;
QUrl actualURL = url;
_isLoaded = false;
// clear the billboard if previously set

6
interface/src/ui/overlays/BillboardOverlay.h
View file

@ -24,6 +24,12 @@ public:
BillboardOverlay();
virtual void render();
// setters
void setURL(const QString& url);
void setScale(float scale) { _scale = scale; }
void setIsFacingAvatar(bool isFacingAvatar) { _isFacingAvatar = isFacingAvatar; }
virtual void setProperties(const QScriptValue& properties);
void setClipFromSource(const QRect& bounds) { _fromImage = bounds; }

73
interface/src/ui/overlays/Overlays.cpp
View file

@ -124,90 +124,59 @@ void Overlays::render3D() {
unsigned int Overlays::addOverlay(const QString& type, const QScriptValue& properties) {
unsigned int thisID = 0;
bool created = false;
bool is3D = false;
Overlay* thisOverlay = NULL;
bool created = true;
if (type == "image") {
thisOverlay = new ImageOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
} else if (type == "text") {
thisOverlay = new TextOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
} else if (type == "cube") {
thisOverlay = new Cube3DOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
is3D = true;
} else if (type == "sphere") {
thisOverlay = new Sphere3DOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
is3D = true;
} else if (type == "circle3d") {
thisOverlay = new Circle3DOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
is3D = true;
} else if (type == "rectangle3d") {
thisOverlay = new Rectangle3DOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
is3D = true;
} else if (type == "line3d") {
thisOverlay = new Line3DOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
is3D = true;
} else if (type == "localvoxels") {
thisOverlay = new LocalVoxelsOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
is3D = true;
} else if (type == "localmodels") {
thisOverlay = new LocalModelsOverlay(Application::getInstance()->getEntityClipboardRenderer());
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
is3D = true;
} else if (type == "model") {
thisOverlay = new ModelOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
is3D = true;
} else if (type == "billboard") {
thisOverlay = new BillboardOverlay();
thisOverlay->init(_parent);
thisOverlay->setProperties(properties);
created = true;
is3D = true;
} else {
created = false;
}
if (created) {
QWriteLocker lock(&_lock);
thisID = _nextOverlayID;
_nextOverlayID++;
if (is3D) {
_overlays3D[thisID] = thisOverlay;
} else {
_overlays2D[thisID] = thisOverlay;
}
thisOverlay->setProperties(properties);
thisID = addOverlay(thisOverlay);
}
return thisID;
}
unsigned int Overlays::addOverlay(Overlay* overlay) {
overlay->init(_parent);
QWriteLocker lock(&_lock);
unsigned int thisID = _nextOverlayID;
_nextOverlayID++;
bool is3D = typeid(*overlay) != typeid(ImageOverlay) && typeid(*overlay) != typeid(TextOverlay);
if (is3D) {
_overlays3D[thisID] = overlay;
} else {
_overlays2D[thisID] = overlay;
}
return thisID;
}
bool Overlays::editOverlay(unsigned int id, const QScriptValue& properties) {
Overlay* thisOverlay = NULL;
QWriteLocker lock(&_lock);

3
interface/src/ui/overlays/Overlays.h
View file

@ -44,6 +44,9 @@ public slots:
/// adds an overlay with the specific properties
unsigned int addOverlay(const QString& type, const QScriptValue& properties);
/// adds an overlay that's already been created
unsigned int addOverlay(Overlay* overlay);
/// edits an overlay updating only the included properties, will return the identified OverlayID in case of
/// successful edit, if the input id is for an unknown overlay this function will have no effect
bool editOverlay(unsigned int id, const QScriptValue& properties);