Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-19 15:43:50 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #4313 from ctrlaltdavid/20355

Browse source 
CR for 20355 - Initial Integration with DDE
This commit is contained in:
Andrew Meadows 2015-02-23 09:47:19 -08:00
commit 0455ccf7be
14 changed files with 106 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
assignment-client/src/Agent.cpp
View file

@ -194,7 +194,7 @@ void Agent::run() {
// setup an Avatar for the script to use
ScriptableAvatar scriptedAvatar(&_scriptEngine);
scriptedAvatar.setForceFaceshiftConnected(true);
scriptedAvatar.setForceFaceTrackerConnected(true);
// call model URL setters with empty URLs so our avatar, if user, will have the default models
scriptedAvatar.setFaceModelURL(QUrl());

11
interface/src/Application.cpp
View file

@ -1624,6 +1624,16 @@ FaceTracker* Application::getActiveFaceTracker() {
(visage->isActive() ? static_cast<FaceTracker*>(visage.data()) : NULL)));
}
void Application::setActiveFaceTracker() {
#ifdef HAVE_FACESHIFT
DependencyManager::get<Faceshift>()->setTCPEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift));
#endif
DependencyManager::get<DdeFaceTracker>()->setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression));
#ifdef HAVE_VISAGE
DependencyManager::get<Visage>()->updateEnabled();
#endif
}
bool Application::exportEntities(const QString& filename, float x, float y, float z, float scale) {
QVector<EntityItem*> entities;
_entities.getTree()->findEntities(AACube(glm::vec3(x / (float)TREE_SCALE,
@ -1741,6 +1751,7 @@ void Application::init() {
// initialize our face trackers after loading the menu settings
DependencyManager::get<Faceshift>()->init();
DependencyManager::get<DdeFaceTracker>()->init();
DependencyManager::get<Visage>()->init();
Leapmotion::init();

2
interface/src/Application.h
View file

@ -366,6 +366,8 @@ public slots:
void notifyPacketVersionMismatch();
void setActiveFaceTracker();
private slots:
void clearDomainOctreeDetails();
void checkFPS();

34
interface/src/Menu.cpp
View file

@ -27,6 +27,7 @@
#include "audio/AudioIOStatsRenderer.h"
#include "audio/AudioScope.h"
#include "avatar/AvatarManager.h"
#include "devices/DdeFaceTracker.h"
#include "devices/Faceshift.h"
#include "devices/RealSense.h"
#include "devices/SixenseManager.h"
@ -357,18 +358,35 @@ Menu::Menu() {
dialogsManager.data(), SLOT(lodTools()));
QMenu* avatarDebugMenu = developerMenu->addMenu("Avatar");
QMenu* faceTrackingMenu = avatarDebugMenu->addMenu("Face Tracking");
{
QActionGroup* faceTrackerGroup = new QActionGroup(avatarDebugMenu);
QAction* noFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::NoFaceTracking,
0, true,
qApp, SLOT(setActiveFaceTracker()));
faceTrackerGroup->addAction(noFaceTracker);
#ifdef HAVE_FACESHIFT
addCheckableActionToQMenuAndActionHash(avatarDebugMenu,
MenuOption::Faceshift,
0,
true,
DependencyManager::get<Faceshift>().data(),
SLOT(setTCPEnabled(bool)));
QAction* faceshiftFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::Faceshift,
0, false,
qApp, SLOT(setActiveFaceTracker()));
faceTrackerGroup->addAction(faceshiftFaceTracker);
#endif
QAction* ddeFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::DDEFaceRegression,
0, false,
qApp, SLOT(setActiveFaceTracker()));
faceTrackerGroup->addAction(ddeFaceTracker);
#ifdef HAVE_VISAGE
addCheckableActionToQMenuAndActionHash(avatarDebugMenu, MenuOption::Visage, 0, false,
DependencyManager::get<Visage>().data(), SLOT(updateEnabled()));
QAction* visageFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::Visage,
0, false,
qApp, SLOT(setActiveFaceTracker()));
faceTrackerGroup->addAction(visageFaceTracker);
#endif
}
addCheckableActionToQMenuAndActionHash(avatarDebugMenu, MenuOption::RenderSkeletonCollisionShapes);
addCheckableActionToQMenuAndActionHash(avatarDebugMenu, MenuOption::RenderHeadCollisionShapes);

10
interface/src/Menu.h
View file

@ -133,13 +133,12 @@ namespace MenuOption {
const QString CollideWithEnvironment = "Collide With World Boundaries";
const QString Collisions = "Collisions";
const QString Console = "Console...";
const QString ControlWithSpeech = "Control With Speech";
const QString CopyAddress = "Copy Address to Clipboard";
const QString CopyPath = "Copy Path to Clipboard";
const QString ControlWithSpeech = "Control With Speech";
const QString DeleteBookmark = "Delete Bookmark...";
const QString DontRenderEntitiesAsScene = "Don't Render Entities as Scene";
const QString DontDoPrecisionPicking = "Don't Do Precision Picking";
const QString DDEFaceRegression = "DDE Face Regression";
const QString DecreaseAvatarSize = "Decrease Avatar Size";
const QString DeleteBookmark = "Delete Bookmark...";
const QString DisableActivityLogger = "Disable Activity Logger";
const QString DisableAutoAdjustLOD = "Disable Automatically Adjusting LOD";
const QString DisableLightEntities = "Disable Light Entities";
@ -152,7 +151,9 @@ namespace MenuOption {
const QString DisplayModelElementChildProxies = "Display Model Element Children";
const QString DisplayModelElementProxy = "Display Model Element Bounds";
const QString DisplayTimingDetails = "Display Timing Details";
const QString DontDoPrecisionPicking = "Don't Do Precision Picking";
const QString DontFadeOnOctreeServerChanges = "Don't Fade In/Out on Octree Server Changes";
const QString DontRenderEntitiesAsScene = "Don't Render Entities as Scene";
const QString EchoLocalAudio = "Echo Local Audio";
const QString EchoServerAudio = "Echo Server Audio";
const QString EditEntitiesHelp = "Edit Entities Help...";
@ -192,6 +193,7 @@ namespace MenuOption {
const QString MuteEnvironment = "Mute Environment";
const QString NetworkSimulator = "Network Simulator...";
const QString NewVoxelCullingMode = "New Voxel Culling Mode";
const QString NoFaceTracking = "None";
const QString ObeyEnvironmentalGravity = "Obey Environmental Gravity";
const QString OctreeStats = "Voxel and Entity Statistics";
const QString OffAxisProjection = "Off-Axis Projection";

13
interface/src/avatar/Head.cpp
View file

@ -80,15 +80,10 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) {
// Only use face trackers when not playing back a recording.
if (!myAvatar->isPlaying()) {
FaceTracker* faceTracker = Application::getInstance()->getActiveFaceTracker();
auto dde = DependencyManager::get<DdeFaceTracker>();
auto faceshift = DependencyManager::get<Faceshift>();
if ((_isFaceshiftConnected = (faceshift == faceTracker))) {
_isFaceTrackerConnected = faceTracker != NULL;
if (_isFaceTrackerConnected) {
_blendshapeCoefficients = faceTracker->getBlendshapeCoefficients();
} else if (dde->isActive()) {
faceTracker = dde.data();
_blendshapeCoefficients = faceTracker->getBlendshapeCoefficients();
}
}
}
// Twist the upper body to follow the rotation of the head, but only do this with my avatar,
// since everyone else will see the full joint rotations for other people.
@ -109,7 +104,7 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) {
_longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f));
}
if (!(_isFaceshiftConnected || billboard)) {
if (!(_isFaceTrackerConnected || billboard)) {
// Update eye saccades
const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f;
const float AVERAGE_SACCADE_INTERVAL = 4.0f;

83
interface/src/devices/DdeFaceTracker.cpp
View file

@ -44,58 +44,40 @@ struct Packet{
};
DdeFaceTracker::DdeFaceTracker() :
_lastReceiveTimestamp(0),
_reset(false),
_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
_rightBlinkIndex(1),
_leftEyeOpenIndex(8),
_rightEyeOpenIndex(9),
_browDownLeftIndex(14),
_browDownRightIndex(15),
_browUpCenterIndex(16),
_browUpLeftIndex(17),
_browUpRightIndex(18),
_mouthSmileLeftIndex(28),
_mouthSmileRightIndex(29),
_jawOpenIndex(21)
DdeFaceTracker(QHostAddress::Any, DDE_FEATURE_POINT_SERVER_PORT)
{
}
DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 port) :
_lastReceiveTimestamp(0),
_reset(false),
_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
_rightBlinkIndex(1),
_leftEyeOpenIndex(8),
_rightEyeOpenIndex(9),
_browDownLeftIndex(14),
_browDownRightIndex(15),
_browUpCenterIndex(16),
_browUpLeftIndex(17),
_browUpRightIndex(18),
_mouthSmileLeftIndex(28),
_mouthSmileRightIndex(29),
_jawOpenIndex(21),
_host(host),
_port(port)
{
_blendshapeCoefficients.resize(NUM_EXPRESSION);
connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams()));
connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError)));
connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), SLOT(socketStateChanged(QAbstractSocket::SocketState)));
bindTo(DDE_FEATURE_POINT_SERVER_PORT);
}
DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 port) :
_lastReceiveTimestamp(0),
_reset(false),
_leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
_rightBlinkIndex(1),
_leftEyeOpenIndex(8),
_rightEyeOpenIndex(9),
_browDownLeftIndex(14),
_browDownRightIndex(15),
_browUpCenterIndex(16),
_browUpLeftIndex(17),
_browUpRightIndex(18),
_mouthSmileLeftIndex(28),
_mouthSmileRightIndex(29),
_jawOpenIndex(21)
{
_blendshapeCoefficients.resize(NUM_EXPRESSION);
connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams()));
connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError)));
connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), SIGNAL(socketStateChanged(QAbstractSocket::SocketState)));
bindTo(host, port);
}
DdeFaceTracker::~DdeFaceTracker() {
if(_udpSocket.isOpen())
if (_udpSocket.isOpen()) {
_udpSocket.close();
}
}
void DdeFaceTracker::init() {
@ -110,15 +92,12 @@ void DdeFaceTracker::update() {
}
void DdeFaceTracker::bindTo(quint16 port) {
bindTo(QHostAddress::Any, port);
}
void DdeFaceTracker::bindTo(const QHostAddress& host, quint16 port) {
if(_udpSocket.isOpen()) {
_udpSocket.close();
void DdeFaceTracker::setEnabled(bool enabled) {
// isOpen() does not work as one might expect on QUdpSocket; don't test isOpen() before closing socket.
_udpSocket.close();
if (enabled) {
_udpSocket.bind(_host, _port);
}
_udpSocket.bind(host, port);
}
bool DdeFaceTracker::isActive() const {
@ -135,7 +114,7 @@ void DdeFaceTracker::socketStateChanged(QAbstractSocket::SocketState socketState
QString state;
switch(socketState) {
case QAbstractSocket::BoundState:
state = "Bounded";
state = "Bound";
break;
case QAbstractSocket::ClosingState:
state = "Closing";
@ -156,7 +135,7 @@ void DdeFaceTracker::socketStateChanged(QAbstractSocket::SocketState socketState
state = "Unconnected";
break;
}
qDebug() << "[Info] DDE Face Tracker Socket: " << socketState;
qDebug() << "[Info] DDE Face Tracker Socket: " << state;
}
void DdeFaceTracker::readPendingDatagrams() {

11
interface/src/devices/DdeFaceTracker.h
View file

@ -28,9 +28,6 @@ public:
void reset();
void update();
//sockets
void bindTo(quint16 port);
void bindTo(const QHostAddress& host, quint16 port);
bool isActive() const;
float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); }
@ -47,7 +44,10 @@ public:
float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); }
float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); }
float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); }
public slots:
void setEnabled(bool enabled);
private slots:
//sockets
@ -59,6 +59,9 @@ private:
DdeFaceTracker();
DdeFaceTracker(const QHostAddress& host, quint16 port);
~DdeFaceTracker();
QHostAddress _host;
quint16 _port;
float getBlendshapeCoefficient(int index) const;
void decodePacket(const QByteArray& buffer);

3
interface/src/devices/Visage.cpp
View file

@ -174,7 +174,8 @@ void Visage::reset() {
void Visage::updateEnabled() {
setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Visage) &&
!(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift) &&
DependencyManager::get<Faceshift>()->isConnectedOrConnecting()));
DependencyManager::get<Faceshift>()->isConnectedOrConnecting()) &&
!Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression));
}
void Visage::setEnabled(bool enabled) {

14
libraries/avatars/src/AvatarData.cpp
View file

@ -44,7 +44,7 @@ AvatarData::AvatarData() :
_handState(0),
_keyState(NO_KEY_DOWN),
_isChatCirclingEnabled(false),
_forceFaceshiftConnected(false),
_forceFaceTrackerConnected(false),
_hasNewJointRotations(true),
_headData(NULL),
_handData(NULL),
@ -136,8 +136,8 @@ QByteArray AvatarData::toByteArray() {
if (!_headData) {
_headData = new HeadData(this);
}
if (_forceFaceshiftConnected) {
_headData->_isFaceshiftConnected = true;
if (_forceFaceTrackerConnected) {
_headData->_isFaceTrackerConnected = true;
}
QByteArray avatarDataByteArray;
@ -191,7 +191,7 @@ QByteArray AvatarData::toByteArray() {
setAtBit(bitItems, HAND_STATE_FINGER_POINTING_BIT);
}
// faceshift state
if (_headData->_isFaceshiftConnected) {
if (_headData->_isFaceTrackerConnected) {
setAtBit(bitItems, IS_FACESHIFT_CONNECTED);
}
if (_isChatCirclingEnabled) {
@ -208,7 +208,7 @@ QByteArray AvatarData::toByteArray() {
}
// If it is connected, pack up the data
if (_headData->_isFaceshiftConnected) {
if (_headData->_isFaceTrackerConnected) {
memcpy(destinationBuffer, &_headData->_leftEyeBlink, sizeof(float));
destinationBuffer += sizeof(float);
@ -417,7 +417,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) {
_handState = getSemiNibbleAt(bitItems, HAND_STATE_START_BIT)
+ (oneAtBit(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0);
_headData->_isFaceshiftConnected = oneAtBit(bitItems, IS_FACESHIFT_CONNECTED);
_headData->_isFaceTrackerConnected = oneAtBit(bitItems, IS_FACESHIFT_CONNECTED);
_isChatCirclingEnabled = oneAtBit(bitItems, IS_CHAT_CIRCLING_ENABLED);
bool hasReferential = oneAtBit(bitItems, HAS_REFERENTIAL);
@ -436,7 +436,7 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) {
}
if (_headData->_isFaceshiftConnected) {
if (_headData->_isFaceTrackerConnected) {
float leftEyeBlink, rightEyeBlink, averageLoudness, browAudioLift;
minPossibleSize += sizeof(leftEyeBlink) + sizeof(rightEyeBlink) + sizeof(averageLoudness) + sizeof(browAudioLift);
minPossibleSize++; // one byte for blendDataSize

4
libraries/avatars/src/AvatarData.h
View file

@ -241,7 +241,7 @@ public:
Q_INVOKABLE void setBlendshape(QString name, float val) { _headData->setBlendshape(name, val); }
void setForceFaceshiftConnected(bool connected) { _forceFaceshiftConnected = connected; }
void setForceFaceTrackerConnected(bool connected) { _forceFaceTrackerConnected = connected; }
// key state
void setKeyState(KeyState s) { _keyState = s; }
@ -357,7 +357,7 @@ protected:
KeyState _keyState;
bool _isChatCirclingEnabled;
bool _forceFaceshiftConnected;
bool _forceFaceTrackerConnected;
bool _hasNewJointRotations; // set in AvatarData, cleared in Avatar
HeadData* _headData;

2
libraries/avatars/src/HeadData.cpp
View file

@ -31,7 +31,7 @@ HeadData::HeadData(AvatarData* owningAvatar) :
_torsoTwist(0.0f),
_lookAtPosition(0.0f, 0.0f, 0.0f),
_audioLoudness(0.0f),
_isFaceshiftConnected(false),
_isFaceTrackerConnected(false),
_leftEyeBlink(0.0f),
_rightEyeBlink(0.0f),
_averageLoudness(0.0f),

2
libraries/avatars/src/HeadData.h
View file

@ -92,7 +92,7 @@ protected:
glm::vec3 _lookAtPosition;
float _audioLoudness;
bool _isFaceshiftConnected;
bool _isFaceTrackerConnected;
float _leftEyeBlink;
float _rightEyeBlink;
float _averageLoudness;

10
libraries/avatars/src/Player.cpp
View file

@ -110,7 +110,7 @@ void Player::startPlaying() {
}
// Fake faceshift connection
_avatar->setForceFaceshiftConnected(true);
_avatar->setForceFaceTrackerConnected(true);
qDebug() << "Recorder::startPlaying()";
setupAudioThread();
@ -136,8 +136,8 @@ void Player::stopPlaying() {
cleanupAudioThread();
_avatar->clearJointsData();
// Turn off fake faceshift connection
_avatar->setForceFaceshiftConnected(false);
// Turn off fake face tracker connection
_avatar->setForceFaceTrackerConnected(false);
if (_useAttachments) {
_avatar->setAttachmentData(_currentContext.attachments);
@ -255,8 +255,8 @@ void Player::play() {
HeadData* head = const_cast<HeadData*>(_avatar->getHeadData());
if (head) {
// Make sure fake faceshift connection doesn't get turned off
_avatar->setForceFaceshiftConnected(true);
// Make sure fake face tracker connection doesn't get turned off
_avatar->setForceFaceTrackerConnected(true);
QVector<float> blendCoef(currentFrame.getBlendshapeCoefficients().size());
for (int i = 0; i < currentFrame.getBlendshapeCoefficients().size(); ++i) {