Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-04-07 03:03:00 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove faceshift

Browse source
This commit is contained in:
Brad Davis 2017-04-28 14:28:03 -07:00
parent 2c2f51689a
commit 5e1bc0d908
27 changed files with 150 additions and 688 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
cmake/externals/faceshift/CMakeLists.txt vendored
View file

@ -1,47 +0,0 @@
set(EXTERNAL_NAME faceshift)
include(ExternalProject)
ExternalProject_Add(
${EXTERNAL_NAME}
URL https://hifi-public.s3.amazonaws.com/dependencies/faceshift.zip
CMAKE_ARGS ${ANDROID_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
BINARY_DIR ${EXTERNAL_PROJECT_PREFIX}/build
LOG_DOWNLOAD 1
LOG_CONFIGURE 1
LOG_BUILD 1
)
# URL_MD5 1bdcb8a0b8d5b1ede434cc41efade41d
# Hide this external target (for ide users)
set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals")
ExternalProject_Get_Property(${EXTERNAL_NAME} INSTALL_DIR)
string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER)
set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include CACHE FILEPATH "Path to Faceshift include directory")
set(LIBRARY_DEBUG_PATH "lib/Debug")
set(LIBRARY_RELEASE_PATH "lib/Release")
if (WIN32)
set(LIBRARY_PREFIX "")
set(LIBRARY_EXT "lib")
# use selected configuration in release path when building on Windows
set(LIBRARY_RELEASE_PATH "$<$<CONFIG:RelWithDebInfo>:build/RelWithDebInfo>")
set(LIBRARY_RELEASE_PATH "${LIBRARY_RELEASE_PATH}$<$<CONFIG:MinSizeRel>:build/MinSizeRel>")
set(LIBRARY_RELEASE_PATH "${LIBRARY_RELEASE_PATH}$<$<OR:$<CONFIG:Release>,$<CONFIG:Debug>>:lib/Release>")
elseif (APPLE)
set(LIBRARY_EXT "a")
set(LIBRARY_PREFIX "lib")
if (CMAKE_GENERATOR STREQUAL "Unix Makefiles")
set(LIBRARY_DEBUG_PATH "build")
set(LIBRARY_RELEASE_PATH "build")
endif ()
endif()
set(${EXTERNAL_NAME_UPPER}_LIBRARY_DEBUG
${INSTALL_DIR}/${LIBRARY_DEBUG_PATH}/${LIBRARY_PREFIX}faceshift.${LIBRARY_EXT} CACHE FILEPATH "Faceshift libraries")
set(${EXTERNAL_NAME_UPPER}_LIBRARY_RELEASE
${INSTALL_DIR}/${LIBRARY_RELEASE_PATH}/${LIBRARY_PREFIX}faceshift.${LIBRARY_EXT} CACHE FILEPATH "Faceshift libraries")

14
cmake/macros/TargetFaceshift.cmake
View file

@ -1,14 +0,0 @@
#
# Copyright 2015 High Fidelity, Inc.
# Created by Bradley Austin Davis on 2015/10/10
#
# Distributed under the Apache License, Version 2.0.
# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
#
macro(TARGET_FACESHIFT)
add_dependency_external_projects(faceshift)
find_package(Faceshift REQUIRED)
target_include_directories(${TARGET_NAME} PRIVATE ${FACESHIFT_INCLUDE_DIRS})
target_link_libraries(${TARGET_NAME} ${FACESHIFT_LIBRARIES})
add_definitions(-DHAVE_FACESHIFT)
endmacro()

26
cmake/modules/FindFaceshift.cmake
View file

@ -1,26 +0,0 @@
#
# FindFaceshift.cmake
#
# Try to find the Faceshift networking library
#
# You must provide a FACESHIFT_ROOT_DIR which contains lib and include directories
#
# Once done this will define
#
# FACESHIFT_FOUND - system found Faceshift
# FACESHIFT_INCLUDE_DIRS - the Faceshift include directory
# FACESHIFT_LIBRARIES - Link this to use Faceshift
#
# Created on 8/30/2013 by Andrzej Kapolka
# Copyright 2013 High Fidelity, Inc.
#
# Distributed under the Apache License, Version 2.0.
# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
#
include(SelectLibraryConfigurations)
select_library_configurations(FACESHIFT)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Faceshift DEFAULT_MSG FACESHIFT_INCLUDE_DIRS FACESHIFT_LIBRARIES)
mark_as_advanced(FACESHIFT_INCLUDE_DIRS FACESHIFT_LIBRARIES FACESHIFT_SEARCH_DIRS)

5
interface/CMakeLists.txt
View file

@ -202,7 +202,6 @@ link_hifi_libraries(
# include the binary directory of render-utils for shader includes
target_include_directories(${TARGET_NAME} PRIVATE "${CMAKE_BINARY_DIR}/libraries/render-utils")
#fixme find a way to express faceshift as a plugin
target_bullet()
target_opengl()
@ -210,10 +209,6 @@ if (NOT ANDROID)
target_glew()
endif ()
if (WIN32 OR APPLE)
target_faceshift()
endif()
# perform standard include and linking for found externals
foreach(EXTERNAL ${OPTIONAL_EXTERNALS})

36
interface/resources/icons/tablet-icons/goto-msg.svg
View file

@ -1,18 +1,18 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 19.2.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 50 50" style="enable-background:new 0 0 50 50;" xml:space="preserve">
<style type="text/css">
.st0{fill:#FFFFFF;}
.st1{fill:#EF3B4E;}
</style>
<g id="Layer_2">
</g>
<g id="Layer_1_1_">
<path class="st0" d="M47.2,41.3l-9.1-9.1c-0.8-0.8-1.9-1.1-3-1l-2.4-2.4c1.8-2.6,2.8-5.7,2.8-9c0-8.9-7.2-16.1-16.1-16.1
S3.3,11,3.3,19.8c0,8.9,7.2,16.1,16.1,16.1c4.1,0,7.8-1.5,10.6-4l2.2,2.2c-0.2,1.1,0.1,2.2,1,3l9.1,9.1c1.4,1.4,3.6,1.4,4.9,0
C48.5,44.9,48.5,42.7,47.2,41.3z M19.4,32.2c-6.8,0-12.3-5.5-12.3-12.3S12.6,7.6,19.4,7.6s12.3,5.5,12.3,12.3
C31.8,26.6,26.2,32.2,19.4,32.2z"/>
</g>
<circle class="st1" cx="43.5" cy="6.5" r="5.9"/>
</svg>
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 19.2.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 50 50" style="enable-background:new 0 0 50 50;" xml:space="preserve">
<style type="text/css">
.st0{fill:#FFFFFF;}
.st1{fill:#EF3B4E;}
</style>
<g id="Layer_2">
</g>
<g id="Layer_1_1_">
<path class="st0" d="M47.2,41.3l-9.1-9.1c-0.8-0.8-1.9-1.1-3-1l-2.4-2.4c1.8-2.6,2.8-5.7,2.8-9c0-8.9-7.2-16.1-16.1-16.1
S3.3,11,3.3,19.8c0,8.9,7.2,16.1,16.1,16.1c4.1,0,7.8-1.5,10.6-4l2.2,2.2c-0.2,1.1,0.1,2.2,1,3l9.1,9.1c1.4,1.4,3.6,1.4,4.9,0
C48.5,44.9,48.5,42.7,47.2,41.3z M19.4,32.2c-6.8,0-12.3-5.5-12.3-12.3S12.6,7.6,19.4,7.6s12.3,5.5,12.3,12.3
C31.8,26.6,26.2,32.2,19.4,32.2z"/>
</g>
<circle class="st1" cx="43.5" cy="6.5" r="5.9"/>
</svg>

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 911 B

After

Width:  |  Height:  |  Size: 929 B

25
interface/src/Application.cpp
View file

@ -137,7 +137,6 @@
#include "CrashHandler.h"
#include "devices/DdeFaceTracker.h"
#include "devices/EyeTracker.h"
#include "devices/Faceshift.h"
#include "devices/Leapmotion.h"
#include "DiscoverabilityManager.h"
#include "GLCanvas.h"
@ -480,7 +479,6 @@ bool setupEssentials(int& argc, char** argv) {
DependencyManager::set<ModelCache>();
DependencyManager::set<ScriptCache>();
DependencyManager::set<SoundCache>();
DependencyManager::set<Faceshift>();
DependencyManager::set<DdeFaceTracker>();
DependencyManager::set<EyeTracker>();
DependencyManager::set<AudioClient>();
@ -1209,10 +1207,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
this->installEventFilter(this);
// initialize our face trackers after loading the menu settings
auto faceshiftTracker = DependencyManager::get<Faceshift>();
faceshiftTracker->init();
connect(faceshiftTracker.data(), &FaceTracker::muteToggled, this, &Application::faceTrackerMuteToggled);
#ifdef HAVE_DDE
auto ddeTracker = DependencyManager::get<DdeFaceTracker>();
ddeTracker->init();
@ -3624,20 +3618,13 @@ ivec2 Application::getMouse() const {
}
FaceTracker* Application::getActiveFaceTracker() {
auto faceshift = DependencyManager::get<Faceshift>();
auto dde = DependencyManager::get<DdeFaceTracker>();
return (dde->isActive() ? static_cast<FaceTracker*>(dde.data()) :
(faceshift->isActive() ? static_cast<FaceTracker*>(faceshift.data()) : nullptr));
return dde->isActive() ? static_cast<FaceTracker*>(dde.data()) : nullptr;
}
FaceTracker* Application::getSelectedFaceTracker() {
FaceTracker* faceTracker = nullptr;
#ifdef HAVE_FACESHIFT
if (Menu::getInstance()->isOptionChecked(MenuOption::Faceshift)) {
faceTracker = DependencyManager::get<Faceshift>().data();
}
#endif
#ifdef HAVE_DDE
if (Menu::getInstance()->isOptionChecked(MenuOption::UseCamera)) {
faceTracker = DependencyManager::get<DdeFaceTracker>().data();
@ -3647,15 +3634,8 @@ FaceTracker* Application::getSelectedFaceTracker() {
}
void Application::setActiveFaceTracker() const {
#if defined(HAVE_FACESHIFT) || defined(HAVE_DDE)
bool isMuted = Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking);
#endif
#ifdef HAVE_FACESHIFT
auto faceshiftTracker = DependencyManager::get<Faceshift>();
faceshiftTracker->setIsMuted(isMuted);
faceshiftTracker->setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift) && !isMuted);
#endif
#ifdef HAVE_DDE
bool isMuted = Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking);
bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::UseCamera);
Menu::getInstance()->getActionForOption(MenuOption::BinaryEyelidControl)->setVisible(isUsingDDE);
Menu::getInstance()->getActionForOption(MenuOption::CoupleEyelids)->setVisible(isUsingDDE);
@ -5131,7 +5111,6 @@ void Application::displaySide(RenderArgs* renderArgs, Camera& theCamera, bool se
}
void Application::resetSensors(bool andReload) {
DependencyManager::get<Faceshift>()->reset();
DependencyManager::get<DdeFaceTracker>()->reset();
DependencyManager::get<EyeTracker>()->reset();
getActiveDisplayPlugin()->resetSensors();

12
interface/src/Menu.cpp
View file

@ -34,7 +34,6 @@
#include "avatar/AvatarManager.h"
#include "AvatarBookmarks.h"
#include "devices/DdeFaceTracker.h"
#include "devices/Faceshift.h"
#include "MainWindow.h"
#include "render/DrawStatus.h"
#include "scripting/MenuScriptingInterface.h"
@ -451,12 +450,6 @@ Menu::Menu() {
qApp, SLOT(setActiveFaceTracker()));
faceTrackerGroup->addAction(noFaceTracker);
#ifdef HAVE_FACESHIFT
QAction* faceshiftFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::Faceshift,
0, false,
qApp, SLOT(setActiveFaceTracker()));
faceTrackerGroup->addAction(faceshiftFaceTracker);
#endif
#ifdef HAVE_DDE
QAction* ddeFaceTracker = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseCamera,
0, true,
@ -477,11 +470,10 @@ Menu::Menu() {
QAction* ddeCalibrate = addActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::CalibrateCamera, 0,
DependencyManager::get<DdeFaceTracker>().data(), SLOT(calibrate()));
ddeCalibrate->setVisible(true); // DDE face tracking is on by default
#endif
#if defined(HAVE_FACESHIFT) || defined(HAVE_DDE)
faceTrackingMenu->addSeparator();
addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::MuteFaceTracking,
Qt::CTRL | Qt::SHIFT | Qt::Key_F, true); // DDE face tracking is on by default
[](bool mute) { FaceTracker::setIsMuted(mute); },
Qt::CTRL | Qt::SHIFT | Qt::Key_F, FaceTracker::isMuted());
addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::AutoMuteAudio, 0, false);
#endif

1
interface/src/Menu.h
View file

@ -105,7 +105,6 @@ namespace MenuOption {
const QString ExpandPaintGLTiming = "Expand /paintGL";
const QString ExpandPhysicsSimulationTiming = "Expand /physics";
const QString ExpandUpdateTiming = "Expand /update";
const QString Faceshift = "Faceshift";
const QString FirstPerson = "First Person";
const QString FivePointCalibration = "5 Point Calibration";
const QString FixGaze = "Fix Gaze (no saccade)";

17
interface/src/avatar/Head.cpp
View file

@ -23,7 +23,6 @@
#include "Util.h"
#include "devices/DdeFaceTracker.h"
#include "devices/EyeTracker.h"
#include "devices/Faceshift.h"
#include <Rig.h>
using namespace std;
@ -209,14 +208,14 @@ void Head::simulate(float deltaTime, bool isMine) {
// use data to update fake Faceshift blendshape coefficients
calculateMouthShapes(deltaTime);
DependencyManager::get<Faceshift>()->updateFakeCoefficients(_leftEyeBlink,
_rightEyeBlink,
_browAudioLift,
_audioJawOpen,
_mouth2,
_mouth3,
_mouth4,
_blendshapeCoefficients);
FaceTracker::updateFakeCoefficients(_leftEyeBlink,
_rightEyeBlink,
_browAudioLift,
_audioJawOpen,
_mouth2,
_mouth3,
_mouth4,
_blendshapeCoefficients);
applyEyelidOffset(getOrientation());

4
interface/src/avatar/MyAvatar.cpp
View file

@ -41,9 +41,9 @@
#include <recording/Clip.h>
#include <recording/Frame.h>
#include <RecordingScriptingInterface.h>
#include <devices/FaceTracker.h>
#include "Application.h"
#include "devices/Faceshift.h"
#include "AvatarManager.h"
#include "AvatarActionHold.h"
#include "Menu.h"
@ -650,7 +650,7 @@ void MyAvatar::updateFromTrackers(float deltaTime) {
}
FaceTracker* tracker = qApp->getActiveFaceTracker();
bool inFacetracker = tracker && !tracker->isMuted();
bool inFacetracker = tracker && !FaceTracker::isMuted();
if (inHmd) {
estimatedPosition = extractTranslation(getHMDSensorMatrix());

3
interface/src/devices/DeviceTracker.cpp
View file

@ -1,7 +1,4 @@
//
// DeviceTracker.cpp
// interface/src/devices
//
// Created by Sam Cake on 6/20/14.
// Copyright 2014 High Fidelity, Inc.
//

3
interface/src/devices/DeviceTracker.h
View file

@ -1,7 +1,4 @@
//
// DeviceTracker.h
// interface/src/devices
//
// Created by Sam Cake on 6/20/14.
// Copyright 2014 High Fidelity, Inc.
//

9
interface/src/devices/EyeTracker.cpp
View file

@ -1,7 +1,4 @@
//
// EyeTracker.cpp
// interface/src/devices
//
// Created by David Rowe on 27 Jul 2015.
// Copyright 2015 High Fidelity, Inc.
//
@ -17,8 +14,8 @@
#include <SharedUtil.h>
#include "InterfaceLogging.h"
#include "OctreeConstants.h"
#include "Logging.h"
#include <OctreeConstants.h>
#ifdef HAVE_IVIEWHMD
char* HIGH_FIDELITY_EYE_TRACKER_CALIBRATION = "HighFidelityEyeTrackerCalibration";
@ -115,7 +112,7 @@ void EyeTracker::processData(smi_CallbackDataStruct* data) {
void EyeTracker::init() {
if (_isInitialized) {
qCWarning(interfaceapp) << "Eye Tracker: Already initialized";
qCWarning(trackers) << "Eye Tracker: Already initialized";
return;
}
}

3
interface/src/devices/EyeTracker.h
View file

@ -1,7 +1,4 @@
//
// EyeTracker.h
// interface/src/devices
//
// Created by David Rowe on 27 Jul 2015.
// Copyright 2015 High Fidelity, Inc.
//

40
interface/src/devices/FaceTracker.cpp
View file

@ -1,7 +1,4 @@
//
// FaceTracker.cpp
// interface/src/devices
//
// Created by Andrzej Kapolka on 4/9/14.
// Copyright 2014 High Fidelity, Inc.
//
@ -9,22 +6,21 @@
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <QTimer>
#include <GLMHelpers.h>
#include "FaceTracker.h"
#include "InterfaceLogging.h"
#include "Menu.h"
#include <QTimer>
#include <GLMHelpers.h>
#include "Logging.h"
//#include "Menu.h"
const int FPS_TIMER_DELAY = 2000; // ms
const int FPS_TIMER_DURATION = 2000; // ms
const float DEFAULT_EYE_DEFLECTION = 0.25f;
Setting::Handle<float> FaceTracker::_eyeDeflection("faceshiftEyeDeflection", DEFAULT_EYE_DEFLECTION);
bool FaceTracker::_isMuted { true };
void FaceTracker::init() {
_isMuted = Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking);
_isInitialized = true; // FaceTracker can be used now
}
@ -101,7 +97,7 @@ void FaceTracker::countFrame() {
}
void FaceTracker::finishFPSTimer() {
qCDebug(interfaceapp) << "Face tracker FPS =" << (float)_frameCount / ((float)FPS_TIMER_DURATION / 1000.0f);
qCDebug(trackers) << "Face tracker FPS =" << (float)_frameCount / ((float)FPS_TIMER_DURATION / 1000.0f);
_isCalculatingFPS = false;
}
@ -113,3 +109,25 @@ void FaceTracker::toggleMute() {
void FaceTracker::setEyeDeflection(float eyeDeflection) {
_eyeDeflection.set(eyeDeflection);
}
void FaceTracker::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp,
float jawOpen, float mouth2, float mouth3, float mouth4, QVector<float>& coefficients) {
const int MMMM_BLENDSHAPE = 34;
const int FUNNEL_BLENDSHAPE = 40;
const int SMILE_LEFT_BLENDSHAPE = 28;
const int SMILE_RIGHT_BLENDSHAPE = 29;
const int MAX_FAKE_BLENDSHAPE = 40; // Largest modified blendshape from above and below
coefficients.resize(std::max((int)coefficients.size(), MAX_FAKE_BLENDSHAPE + 1));
qFill(coefficients.begin(), coefficients.end(), 0.0f);
coefficients[_leftBlinkIndex] = leftBlink;
coefficients[_rightBlinkIndex] = rightBlink;
coefficients[_browUpCenterIndex] = browUp;
coefficients[_browUpLeftIndex] = browUp;
coefficients[_browUpRightIndex] = browUp;
coefficients[_jawOpenIndex] = jawOpen;
coefficients[SMILE_LEFT_BLENDSHAPE] = coefficients[SMILE_RIGHT_BLENDSHAPE] = mouth4;
coefficients[MMMM_BLENDSHAPE] = mouth2;
coefficients[FUNNEL_BLENDSHAPE] = mouth3;
}

38
interface/src/devices/FaceTracker.h
View file

@ -1,7 +1,4 @@
//
// FaceTracker.h
// interface/src/devices
//
// Created by Andrzej Kapolka on 4/9/14.
// Copyright 2014 High Fidelity, Inc.
//
@ -20,7 +17,7 @@
#include <SettingHandle.h>
/// Base class for face trackers (Faceshift, DDE).
/// Base class for face trackers (DDE, BinaryVR).
class FaceTracker : public QObject {
Q_OBJECT
@ -45,12 +42,21 @@ public:
const QVector<float>& getBlendshapeCoefficients() const;
float getBlendshapeCoefficient(int index) const;
bool isMuted() const { return _isMuted; }
void setIsMuted(bool isMuted) { _isMuted = isMuted; }
static bool isMuted() { return _isMuted; }
static void setIsMuted(bool isMuted) { _isMuted = isMuted; }
static float getEyeDeflection() { return _eyeDeflection.get(); }
static void setEyeDeflection(float eyeDeflection);
static void updateFakeCoefficients(float leftBlink,
float rightBlink,
float browUp,
float jawOpen,
float mouth2,
float mouth3,
float mouth4,
QVector<float>& coefficients);
signals:
void muteToggled();
@ -63,7 +69,7 @@ protected:
virtual ~FaceTracker() {};
bool _isInitialized = false;
bool _isMuted = true;
static bool _isMuted;
glm::vec3 _headTranslation = glm::vec3(0.0f);
glm::quat _headRotation = glm::quat();
@ -84,6 +90,24 @@ private:
bool _isCalculatingFPS = false;
int _frameCount = 0;
// see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
static const int _leftBlinkIndex = 0;
static const int _rightBlinkIndex = 1;
static const int _leftEyeOpenIndex = 8;
static const int _rightEyeOpenIndex = 9;
// Brows
static const int _browDownLeftIndex = 14;
static const int _browDownRightIndex = 15;
static const int _browUpCenterIndex = 16;
static const int _browUpLeftIndex = 17;
static const int _browUpRightIndex = 18;
static const int _mouthSmileLeftIndex = 28;
static const int _mouthSmileRightIndex = 29;
static const int _jawOpenIndex = 21;
static Setting::Handle<float> _eyeDeflection;
};

310
interface/src/devices/Faceshift.cpp
View file

@ -1,310 +0,0 @@
//
// Faceshift.cpp
// interface/src/devices
//
// Created by Andrzej Kapolka on 9/3/13.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <QTimer>
#include <GLMHelpers.h>
#include <NumericalConstants.h>
#include <PerfStat.h>
#include "Faceshift.h"
#include "Menu.h"
#include "Util.h"
#include "InterfaceLogging.h"
#ifdef HAVE_FACESHIFT
using namespace fs;
#endif
using namespace std;
const QString DEFAULT_FACESHIFT_HOSTNAME = "localhost";
const quint16 FACESHIFT_PORT = 33433;
Faceshift::Faceshift() :
_hostname("faceshiftHostname", DEFAULT_FACESHIFT_HOSTNAME)
{
#ifdef HAVE_FACESHIFT
connect(&_tcpSocket, SIGNAL(connected()), SLOT(noteConnected()));
connect(&_tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError)));
connect(&_tcpSocket, SIGNAL(readyRead()), SLOT(readFromSocket()));
connect(&_tcpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), SIGNAL(connectionStateChanged()));
connect(&_tcpSocket, SIGNAL(disconnected()), SLOT(noteDisconnected()));
connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams()));
_udpSocket.bind(FACESHIFT_PORT);
#endif
}
#ifdef HAVE_FACESHIFT
void Faceshift::init() {
FaceTracker::init();
setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift) && !_isMuted);
}
void Faceshift::update(float deltaTime) {
if (!isActive()) {
return;
}
FaceTracker::update(deltaTime);
// get the euler angles relative to the window
glm::vec3 eulers = glm::degrees(safeEulerAngles(_headRotation * glm::quat(glm::radians(glm::vec3(
(_eyeGazeLeftPitch + _eyeGazeRightPitch) / 2.0f, (_eyeGazeLeftYaw + _eyeGazeRightYaw) / 2.0f, 0.0f)))));
// compute and subtract the long term average
const float LONG_TERM_AVERAGE_SMOOTHING = 0.999f;
if (!_longTermAverageInitialized) {
_longTermAverageEyePitch = eulers.x;
_longTermAverageEyeYaw = eulers.y;
_longTermAverageInitialized = true;
} else {
_longTermAverageEyePitch = glm::mix(eulers.x, _longTermAverageEyePitch, LONG_TERM_AVERAGE_SMOOTHING);
_longTermAverageEyeYaw = glm::mix(eulers.y, _longTermAverageEyeYaw, LONG_TERM_AVERAGE_SMOOTHING);
}
_estimatedEyePitch = eulers.x - _longTermAverageEyePitch;
_estimatedEyeYaw = eulers.y - _longTermAverageEyeYaw;
}
void Faceshift::reset() {
if (_tcpSocket.state() == QAbstractSocket::ConnectedState) {
qCDebug(interfaceapp, "Faceshift: Reset");
FaceTracker::reset();
string message;
fsBinaryStream::encode_message(message, fsMsgCalibrateNeutral());
send(message);
}
_longTermAverageInitialized = false;
}
bool Faceshift::isActive() const {
const quint64 ACTIVE_TIMEOUT_USECS = 1000000;
return (usecTimestampNow() - _lastReceiveTimestamp) < ACTIVE_TIMEOUT_USECS;
}
bool Faceshift::isTracking() const {
return isActive() && _tracking;
}
#endif
bool Faceshift::isConnectedOrConnecting() const {
return _tcpSocket.state() == QAbstractSocket::ConnectedState ||
(_tcpRetryCount == 0 && _tcpSocket.state() != QAbstractSocket::UnconnectedState);
}
void Faceshift::updateFakeCoefficients(float leftBlink, float rightBlink, float browUp,
float jawOpen, float mouth2, float mouth3, float mouth4, QVector<float>& coefficients) const {
const int MMMM_BLENDSHAPE = 34;
const int FUNNEL_BLENDSHAPE = 40;
const int SMILE_LEFT_BLENDSHAPE = 28;
const int SMILE_RIGHT_BLENDSHAPE = 29;
const int MAX_FAKE_BLENDSHAPE = 40; // Largest modified blendshape from above and below
coefficients.resize(max((int)coefficients.size(), MAX_FAKE_BLENDSHAPE + 1));
qFill(coefficients.begin(), coefficients.end(), 0.0f);
coefficients[_leftBlinkIndex] = leftBlink;
coefficients[_rightBlinkIndex] = rightBlink;
coefficients[_browUpCenterIndex] = browUp;
coefficients[_browUpLeftIndex] = browUp;
coefficients[_browUpRightIndex] = browUp;
coefficients[_jawOpenIndex] = jawOpen;
coefficients[SMILE_LEFT_BLENDSHAPE] = coefficients[SMILE_RIGHT_BLENDSHAPE] = mouth4;
coefficients[MMMM_BLENDSHAPE] = mouth2;
coefficients[FUNNEL_BLENDSHAPE] = mouth3;
}
void Faceshift::setEnabled(bool enabled) {
// Don't enable until have explicitly initialized
if (!_isInitialized) {
return;
}
#ifdef HAVE_FACESHIFT
if ((_tcpEnabled = enabled)) {
connectSocket();
} else {
qCDebug(interfaceapp, "Faceshift: Disconnecting...");
_tcpSocket.disconnectFromHost();
}
#endif
}
void Faceshift::connectSocket() {
if (_tcpEnabled) {
if (!_tcpRetryCount) {
qCDebug(interfaceapp, "Faceshift: Connecting...");
}
_tcpSocket.connectToHost(_hostname.get(), FACESHIFT_PORT);
_tracking = false;
}
}
void Faceshift::noteConnected() {
#ifdef HAVE_FACESHIFT
qCDebug(interfaceapp, "Faceshift: Connected");
// request the list of blendshape names
string message;
fsBinaryStream::encode_message(message, fsMsgSendBlendshapeNames());
send(message);
#endif
}
void Faceshift::noteDisconnected() {
#ifdef HAVE_FACESHIFT
qCDebug(interfaceapp, "Faceshift: Disconnected");
#endif
}
void Faceshift::noteError(QAbstractSocket::SocketError error) {
if (!_tcpRetryCount) {
// Only spam log with fail to connect the first time, so that we can keep waiting for server
qCWarning(interfaceapp) << "Faceshift: " << _tcpSocket.errorString();
}
// retry connection after a 2 second delay
if (_tcpEnabled) {
_tcpRetryCount++;
QTimer::singleShot(2000, this, SLOT(connectSocket()));
}
}
void Faceshift::readPendingDatagrams() {
QByteArray buffer;
while (_udpSocket.hasPendingDatagrams()) {
buffer.resize(_udpSocket.pendingDatagramSize());
_udpSocket.readDatagram(buffer.data(), buffer.size());
receive(buffer);
}
}
void Faceshift::readFromSocket() {
receive(_tcpSocket.readAll());
}
void Faceshift::send(const std::string& message) {
_tcpSocket.write(message.data(), message.size());
}
void Faceshift::receive(const QByteArray& buffer) {
#ifdef HAVE_FACESHIFT
_lastReceiveTimestamp = usecTimestampNow();
_stream.received(buffer.size(), buffer.constData());
fsMsgPtr msg;
for (fsMsgPtr msg; (msg = _stream.get_message()); ) {
switch (msg->id()) {
case fsMsg::MSG_OUT_TRACKING_STATE: {
const fsTrackingData& data = static_pointer_cast<fsMsgTrackingState>(msg)->tracking_data();
if ((_tracking = data.m_trackingSuccessful)) {
glm::quat newRotation = glm::quat(data.m_headRotation.w, -data.m_headRotation.x,
data.m_headRotation.y, -data.m_headRotation.z);
// Compute angular velocity of the head
glm::quat r = glm::normalize(newRotation * glm::inverse(_headRotation));
float theta = 2 * acos(r.w);
if (theta > EPSILON) {
float rMag = glm::length(glm::vec3(r.x, r.y, r.z));
_headAngularVelocity = theta / _averageFrameTime * glm::vec3(r.x, r.y, r.z) / rMag;
} else {
_headAngularVelocity = glm::vec3(0,0,0);
}
const float ANGULAR_VELOCITY_FILTER_STRENGTH = 0.3f;
_headRotation = safeMix(_headRotation, newRotation, glm::clamp(glm::length(_headAngularVelocity) *
ANGULAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f));
const float TRANSLATION_SCALE = 0.02f;
glm::vec3 newHeadTranslation = glm::vec3(data.m_headTranslation.x, data.m_headTranslation.y,
-data.m_headTranslation.z) * TRANSLATION_SCALE;
_headLinearVelocity = (newHeadTranslation - _lastHeadTranslation) / _averageFrameTime;
const float LINEAR_VELOCITY_FILTER_STRENGTH = 0.3f;
float velocityFilter = glm::clamp(1.0f - glm::length(_headLinearVelocity) *
LINEAR_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);
_filteredHeadTranslation = velocityFilter * _filteredHeadTranslation + (1.0f - velocityFilter) * newHeadTranslation;
_lastHeadTranslation = newHeadTranslation;
_headTranslation = _filteredHeadTranslation;
_eyeGazeLeftPitch = -data.m_eyeGazeLeftPitch;
_eyeGazeLeftYaw = data.m_eyeGazeLeftYaw;
_eyeGazeRightPitch = -data.m_eyeGazeRightPitch;
_eyeGazeRightYaw = data.m_eyeGazeRightYaw;
_blendshapeCoefficients = QVector<float>::fromStdVector(data.m_coeffs);
const float FRAME_AVERAGING_FACTOR = 0.99f;
quint64 usecsNow = usecTimestampNow();
if (_lastMessageReceived != 0) {
_averageFrameTime = FRAME_AVERAGING_FACTOR * _averageFrameTime +
(1.0f - FRAME_AVERAGING_FACTOR) * (float)(usecsNow - _lastMessageReceived) / 1000000.0f;
}
_lastMessageReceived = usecsNow;
}
break;
}
case fsMsg::MSG_OUT_BLENDSHAPE_NAMES: {
const vector<string>& names = static_pointer_cast<fsMsgBlendshapeNames>(msg)->blendshape_names();
for (int i = 0; i < (int)names.size(); i++) {
if (names[i] == "EyeBlink_L") {
_leftBlinkIndex = i;
} else if (names[i] == "EyeBlink_R") {
_rightBlinkIndex = i;
} else if (names[i] == "EyeOpen_L") {
_leftEyeOpenIndex = i;
} else if (names[i] == "EyeOpen_R") {
_rightEyeOpenIndex = i;
} else if (names[i] == "BrowsD_L") {
_browDownLeftIndex = i;
} else if (names[i] == "BrowsD_R") {
_browDownRightIndex = i;
} else if (names[i] == "BrowsU_C") {
_browUpCenterIndex = i;
} else if (names[i] == "BrowsU_L") {
_browUpLeftIndex = i;
} else if (names[i] == "BrowsU_R") {
_browUpRightIndex = i;
} else if (names[i] == "JawOpen") {
_jawOpenIndex = i;
} else if (names[i] == "MouthSmile_L") {
_mouthSmileLeftIndex = i;
} else if (names[i] == "MouthSmile_R") {
_mouthSmileRightIndex = i;
}
}
break;
}
default:
break;
}
}
#endif
FaceTracker::countFrame();
}
void Faceshift::setHostname(const QString& hostname) {
_hostname.set(hostname);
}

155
interface/src/devices/Faceshift.h
View file

@ -1,155 +0,0 @@
//
// Faceshift.h
// interface/src/devices
//
// Created by Andrzej Kapolka on 9/3/13.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_Faceshift_h
#define hifi_Faceshift_h
#include <QTcpSocket>
#include <QUdpSocket>
#ifdef HAVE_FACESHIFT
#include <fsbinarystream.h>
#endif
#include <DependencyManager.h>
#include <SettingHandle.h>
#include "FaceTracker.h"
const float STARTING_FACESHIFT_FRAME_TIME = 0.033f;
/// Handles interaction with the Faceshift software, which provides head position/orientation and facial features.
class Faceshift : public FaceTracker, public Dependency {
Q_OBJECT
SINGLETON_DEPENDENCY
public:
#ifdef HAVE_FACESHIFT
// If we don't have faceshift, use the base class' methods
virtual void init() override;
virtual void update(float deltaTime) override;
virtual void reset() override;
virtual bool isActive() const override;
virtual bool isTracking() const override;
#endif
bool isConnectedOrConnecting() const;
const glm::vec3& getHeadAngularVelocity() const { return _headAngularVelocity; }
// these pitch/yaw angles are in degrees
float getEyeGazeLeftPitch() const { return _eyeGazeLeftPitch; }
float getEyeGazeLeftYaw() const { return _eyeGazeLeftYaw; }
float getEyeGazeRightPitch() const { return _eyeGazeRightPitch; }
float getEyeGazeRightYaw() const { return _eyeGazeRightYaw; }
float getLeftBlink() const { return getBlendshapeCoefficient(_leftBlinkIndex); }
float getRightBlink() const { return getBlendshapeCoefficient(_rightBlinkIndex); }
float getLeftEyeOpen() const { return getBlendshapeCoefficient(_leftEyeOpenIndex); }
float getRightEyeOpen() const { return getBlendshapeCoefficient(_rightEyeOpenIndex); }
float getBrowDownLeft() const { return getBlendshapeCoefficient(_browDownLeftIndex); }
float getBrowDownRight() const { return getBlendshapeCoefficient(_browDownRightIndex); }
float getBrowUpCenter() const { return getBlendshapeCoefficient(_browUpCenterIndex); }
float getBrowUpLeft() const { return getBlendshapeCoefficient(_browUpLeftIndex); }
float getBrowUpRight() const { return getBlendshapeCoefficient(_browUpRightIndex); }
float getMouthSize() const { return getBlendshapeCoefficient(_jawOpenIndex); }
float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); }
float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); }
QString getHostname() { return _hostname.get(); }
void setHostname(const QString& hostname);
void updateFakeCoefficients(float leftBlink,
float rightBlink,
float browUp,
float jawOpen,
float mouth2,
float mouth3,
float mouth4,
QVector<float>& coefficients) const;
signals:
void connectionStateChanged();
public slots:
void setEnabled(bool enabled) override;
private slots:
void connectSocket();
void noteConnected();
void noteError(QAbstractSocket::SocketError error);
void readPendingDatagrams();
void readFromSocket();
void noteDisconnected();
private:
Faceshift();
virtual ~Faceshift() {}
void send(const std::string& message);
void receive(const QByteArray& buffer);
QTcpSocket _tcpSocket;
QUdpSocket _udpSocket;
#ifdef HAVE_FACESHIFT
fs::fsBinaryStream _stream;
#endif
bool _tcpEnabled = true;
int _tcpRetryCount = 0;
bool _tracking = false;
quint64 _lastReceiveTimestamp = 0;
quint64 _lastMessageReceived = 0;
float _averageFrameTime = STARTING_FACESHIFT_FRAME_TIME;
glm::vec3 _headAngularVelocity = glm::vec3(0.0f);
glm::vec3 _headLinearVelocity = glm::vec3(0.0f);
glm::vec3 _lastHeadTranslation = glm::vec3(0.0f);
glm::vec3 _filteredHeadTranslation = glm::vec3(0.0f);
// degrees
float _eyeGazeLeftPitch = 0.0f;
float _eyeGazeLeftYaw = 0.0f;
float _eyeGazeRightPitch = 0.0f;
float _eyeGazeRightYaw = 0.0f;
// degrees
float _longTermAverageEyePitch = 0.0f;
float _longTermAverageEyeYaw = 0.0f;
bool _longTermAverageInitialized = false;
Setting::Handle<QString> _hostname;
// see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
int _leftBlinkIndex = 0;
int _rightBlinkIndex = 1;
int _leftEyeOpenIndex = 8;
int _rightEyeOpenIndex = 9;
// Brows
int _browDownLeftIndex = 14;
int _browDownRightIndex = 15;
int _browUpCenterIndex = 16;
int _browUpLeftIndex = 17;
int _browUpRightIndex = 18;
int _mouthSmileLeftIndex = 28;
int _mouthSmileRightIndex = 29;
int _jawOpenIndex = 21;
};
#endif // hifi_Faceshift_h

11
interface/src/devices/Logging.cpp Normal file
View file

@ -0,0 +1,11 @@
//
// Created by Bradley Austin Davis on 2017/04/25
// Copyright 2013-2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "Logging.h"
Q_LOGGING_CATEGORY(trackers, "hifi.trackers")

16
interface/src/devices/Logging.h Normal file
View file

@ -0,0 +1,16 @@
//
// Created by Bradley Austin Davis on 2017/04/25
// Copyright 2013-2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_TrackersLogging_h
#define hifi_TrackersLogging_h
#include <QtCore/QLoggingCategory>
Q_DECLARE_LOGGING_CATEGORY(trackers)
#endif // hifi_TrackersLogging_h

5
interface/src/devices/MotionTracker.cpp
View file

@ -1,7 +1,4 @@
//
// MotionTracker.cpp
// interface/src/devices
//
// Created by Sam Cake on 6/20/14.
// Copyright 2014 High Fidelity, Inc.
//
@ -10,8 +7,6 @@
//
#include "MotionTracker.h"
#include "GLMHelpers.h"
// glm::mult(mat43, mat43) just the composition of the 2 matrices assuming they are in fact mat44 with the last raw = { 0, 0, 0, 1 }
namespace glm {

18
interface/src/devices/MotionTracker.h
View file

@ -1,7 +1,4 @@
//
// MotionTracker.h
// interface/src/devices
//
// Created by Sam Cake on 6/20/14.
// Copyright 2014 High Fidelity, Inc.
//
@ -14,20 +11,7 @@
#include "DeviceTracker.h"
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-compare"
#endif
#include <glm/glm.hpp>
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif
#include <glm/gtc/quaternion.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <GLMHelpers.h>
/// Base class for device trackers.
class MotionTracker : public DeviceTracker {

8
interface/src/ui/PreferencesDialog.cpp
View file

@ -11,7 +11,6 @@
#include <AudioClient.h>
#include <avatar/AvatarManager.h>
#include <devices/DdeFaceTracker.h>
#include <devices/Faceshift.h>
#include <NetworkingConstants.h>
#include <ScriptEngines.h>
#include <OffscreenUi.h>
@ -207,13 +206,6 @@ void setupPreferences() {
auto setter = [](float value) { FaceTracker::setEyeDeflection(value); };
preferences->addPreference(new SliderPreference(AVATAR_TUNING, "Face tracker eye deflection", getter, setter));
}
{
auto getter = []()->QString { return DependencyManager::get<Faceshift>()->getHostname(); };
auto setter = [](const QString& value) { DependencyManager::get<Faceshift>()->setHostname(value); };
auto preference = new EditPreference(AVATAR_TUNING, "Faceshift hostname", getter, setter);
preference->setPlaceholderText("localhost");
preferences->addPreference(preference);
}
{
auto getter = [=]()->QString { return myAvatar->getAnimGraphOverrideUrl().toString(); };
auto setter = [=](const QString& value) { myAvatar->setAnimGraphOverrideUrl(QUrl(value)); };

6
libraries/avatars/src/AvatarData.cpp
View file

@ -393,9 +393,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
if (isFingerPointing) {
setAtBit(flags, HAND_STATE_FINGER_POINTING_BIT);
}
// faceshift state
// face tracker state
if (_headData->_isFaceTrackerConnected) {
setAtBit(flags, IS_FACESHIFT_CONNECTED);
setAtBit(flags, IS_FACE_TRACKER_CONNECTED);
}
// eye tracker state
if (_headData->_isEyeTrackerConnected) {
@ -883,7 +883,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) {
auto newHandState = getSemiNibbleAt(bitItems, HAND_STATE_START_BIT)
+ (oneAtBit(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0);
auto newFaceTrackerConnected = oneAtBit(bitItems, IS_FACESHIFT_CONNECTED);
auto newFaceTrackerConnected = oneAtBit(bitItems, IS_FACE_TRACKER_CONNECTED);
auto newEyeTrackerConnected = oneAtBit(bitItems, IS_EYE_TRACKER_CONNECTED);
bool keyStateChanged = (_keyState != newKeyState);

4
libraries/avatars/src/AvatarData.h
View file

@ -99,7 +99,7 @@ const quint32 AVATAR_MOTION_SCRIPTABLE_BITS =
// Referential Data - R is found in the 7th bit
const int KEY_STATE_START_BIT = 0; // 1st and 2nd bits
const int HAND_STATE_START_BIT = 2; // 3rd and 4th bits
const int IS_FACESHIFT_CONNECTED = 4; // 5th bit
const int IS_FACE_TRACKER_CONNECTED = 4; // 5th bit
const int IS_EYE_TRACKER_CONNECTED = 5; // 6th bit (was CHAT_CIRCLING)
const int HAS_REFERENTIAL = 6; // 7th bit
const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit
@ -218,7 +218,7 @@ namespace AvatarDataPacket {
} PACKED_END;
const size_t AVATAR_LOCAL_POSITION_SIZE = 12;
// only present if IS_FACESHIFT_CONNECTED flag is set in AvatarInfo.flags
// only present if IS_FACE_TRACKER_CONNECTED flag is set in AvatarInfo.flags
PACKED_BEGIN struct FaceTrackerInfo {
float leftEyeBlink;
float rightEyeBlink;

12
libraries/ui/src/ui/Menu.cpp
View file

@ -223,6 +223,18 @@ QAction* Menu::addCheckableActionToQMenuAndActionHash(MenuWrapper* destinationMe
return action;
}
QAction* Menu::addCheckableActionToQMenuAndActionHash(MenuWrapper* destinationMenu,
const QString& actionName,
const std::function<void(bool)>& handler,
const QKeySequence& shortcut,
const bool checked,
int menuItemLocation,
const QString& grouping) {
auto action = addCheckableActionToQMenuAndActionHash(destinationMenu, actionName, shortcut, checked, nullptr, nullptr, menuItemLocation, grouping);
connect(action, &QAction::triggered, handler);
return action;
}
void Menu::removeAction(MenuWrapper* menu, const QString& actionName) {
auto action = _actionHash.value(actionName);
menu->removeAction(action);

10
libraries/ui/src/ui/Menu.h
View file

@ -9,6 +9,8 @@
#ifndef hifi_ui_Menu_h
#define hifi_ui_Menu_h
#include <functional>
#include <QtCore/QDir>
#include <QtCore/QPointer>
#include <QtCore/QStandardPaths>
@ -90,6 +92,14 @@ public:
int menuItemLocation = UNSPECIFIED_POSITION,
const QString& grouping = QString());
QAction* addCheckableActionToQMenuAndActionHash(MenuWrapper* destinationMenu,
const QString& actionName,
const std::function<void(bool)>& handler,
const QKeySequence& shortcut = 0,
const bool checked = false,
int menuItemLocation = UNSPECIFIED_POSITION,
const QString& grouping = QString());
void removeAction(MenuWrapper* menu, const QString& actionName);
public slots: