From 78e15e021268d5bc3ed3a3f9b9dc1f3779e51d08 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 11 Feb 2014 16:03:23 -0800 Subject: [PATCH 01/21] Starting on Visage support. --- cmake/modules/FindVisage.cmake | 0 interface/external/visage/readme.txt | 8 ++++++++ 2 files changed, 8 insertions(+) create mode 100644 cmake/modules/FindVisage.cmake create mode 100644 interface/external/visage/readme.txt diff --git a/cmake/modules/FindVisage.cmake b/cmake/modules/FindVisage.cmake new file mode 100644 index 0000000000..e69de29bb2 diff --git a/interface/external/visage/readme.txt b/interface/external/visage/readme.txt new file mode 100644 index 0000000000..fe2e81f87b --- /dev/null +++ b/interface/external/visage/readme.txt @@ -0,0 +1,8 @@ + +Instructions for adding the Visage driver to Interface +Andrzej Kapolka, February 11, 2014 + +1. Copy the Visage sdk folders (lib, include) into the interface/external/visage folder. This readme.txt should be there as well. + +2. Delete your build directory, run cmake and build, and you should be all set. + From 25066f8b4f81f2d11427d996896c76452a343a00 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 11 Feb 2014 16:44:25 -0800 Subject: [PATCH 02/21] Starting on Visage integration. --- cmake/modules/FindVisage.cmake | 44 ++++++++++++++++++++++++++++ interface/CMakeLists.txt | 14 ++++++++- interface/external/visage/readme.txt | 10 +++++-- interface/src/Application.h | 2 ++ interface/src/devices/Visage.cpp | 36 +++++++++++++++++++++++ interface/src/devices/Visage.h | 28 ++++++++++++++++++ 6 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 interface/src/devices/Visage.cpp create mode 100644 interface/src/devices/Visage.h diff --git a/cmake/modules/FindVisage.cmake b/cmake/modules/FindVisage.cmake index e69de29bb2..a8d1acc30b 100644 --- a/cmake/modules/FindVisage.cmake +++ b/cmake/modules/FindVisage.cmake @@ -0,0 +1,44 @@ +# Try to find the Visage controller library +# +# You must provide a VISAGE_ROOT_DIR which contains lib and include directories +# +# Once done this will define +# +# VISAGE_FOUND - system found Visage +# VISAGE_INCLUDE_DIRS - the Visage include directory +# VISAGE_LIBRARIES - Link this to use Visage +# +# Created on 2/11/2014 by Andrzej Kapolka +# Copyright (c) 2014 High Fidelity +# + +if (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) + # in cache already + set(VISAGE_FOUND TRUE) +else (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) + find_path(VISAGE_INCLUDE_DIRS VisageTracker2.h ${VISAGE_ROOT_DIR}/include) + + if (APPLE) + find_library(VISAGE_LIBRARIES libvscore.a ${VISAGE_ROOT_DIR}/lib) + elseif (WIN32) + find_library(VISAGE_LIBRARIES vscore.dll ${VISAGE_ROOT_DIR}/lib) + endif () + + if (VISAGE_INCLUDE_DIRS AND VISAGE_LIBRARIES) + set(VISAGE_FOUND TRUE) + endif (VISAGE_INCLUDE_DIRS AND VISAGE_LIBRARIES) + + if (VISAGE_FOUND) + if (NOT VISAGE_FIND_QUIETLY) + message(STATUS "Found Visage: ${VISAGE_LIBRARIES}") + endif (NOT VISAGE_FIND_QUIETLY) + else (VISAGE_FOUND) + if (VISAGE_FIND_REQUIRED) + message(FATAL_ERROR "Could not find Visage") + endif (VISAGE_FIND_REQUIRED) + endif (VISAGE_FOUND) + + # show the VISAGE_INCLUDE_DIRS and VISAGE_LIBRARIES variables only in the advanced view + mark_as_advanced(VISAGE_INCLUDE_DIRS VISAGE_LIBRARIES) + +endif (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 6af6ed478d..188e7a6995 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -11,6 +11,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../cmake set(FACESHIFT_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/faceshift) set(LIBOVR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/LibOVR) set(SIXENSE_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/Sixense) +set(VISAGE_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/visage) if (DEFINED ENV{JOB_ID}) set(BUILD_SEQ $ENV{JOB_ID}) @@ -132,9 +133,10 @@ find_package(Faceshift) find_package(GLM REQUIRED) find_package(LibOVR) find_package(Sixense) +find_package(Visage) find_package(ZLIB) -# likewise with Sixense library for Razer Hydra +# include the Sixense library for Razer Hydra if available if (SIXENSE_FOUND AND NOT DISABLE_SIXENSE) add_definitions(-DHAVE_SIXENSE) include_directories(SYSTEM ${SIXENSE_INCLUDE_DIRS}) @@ -144,6 +146,16 @@ if (SIXENSE_FOUND AND NOT DISABLE_SIXENSE) target_link_libraries(${TARGET_NAME} ${SIXENSE_LIBRARIES}) endif (SIXENSE_FOUND AND NOT DISABLE_SIXENSE) +# likewise with Visage library for webcam feature tracking +if (VISAGE_FOUND AND NOT DISABLE_VISAGE) + add_definitions(-DHAVE_VISAGE) + include_directories(SYSTEM ${VISAGE_INCLUDE_DIRS}) + if (APPLE OR UNIX) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${VISAGE_INCLUDE_DIRS}") + endif (APPLE OR UNIX) + target_link_libraries(${TARGET_NAME} ${VISAGE_LIBRARIES}) +endif (VISAGE_FOUND AND NOT DISABLE_VISAGE) + # and with LibOVR for Oculus Rift if (LIBOVR_FOUND AND NOT DISABLE_LIBOVR) add_definitions(-DHAVE_LIBOVR) diff --git a/interface/external/visage/readme.txt b/interface/external/visage/readme.txt index fe2e81f87b..dfb3da3969 100644 --- a/interface/external/visage/readme.txt +++ b/interface/external/visage/readme.txt @@ -2,7 +2,13 @@ Instructions for adding the Visage driver to Interface Andrzej Kapolka, February 11, 2014 -1. Copy the Visage sdk folders (lib, include) into the interface/external/visage folder. This readme.txt should be there as well. +1. Copy the Visage sdk folders (lib, include) into the interface/external/visage folder. + This readme.txt should be there as well. -2. Delete your build directory, run cmake and build, and you should be all set. +2. Copy the Visage configuration data folder (visageSDK-MacOS/Samples/MacOSX/data/) to interface/resources/visage + (i.e., so that interface/resources/visage/candide3.wfm is accessible) + +3. Copy the Visage license file to interface/resources/visage/license.vlc. + +4. Delete your build directory, run cmake and build, and you should be all set. diff --git a/interface/src/Application.h b/interface/src/Application.h index ff6e08758b..b2af223edc 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -55,6 +55,7 @@ #include "avatar/Profile.h" #include "devices/Faceshift.h" #include "devices/SixenseManager.h" +#include "devices/Visage.h" #include "renderer/AmbientOcclusionEffect.h" #include "renderer/GeometryCache.h" #include "renderer/GlowEffect.h" @@ -378,6 +379,7 @@ private: Profile _profile; // The data-server linked profile for this user Faceshift _faceshift; + Visage _visage; SixenseManager _sixenseManager; QStringList _activeScripts; diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp new file mode 100644 index 0000000000..4b986d7db4 --- /dev/null +++ b/interface/src/devices/Visage.cpp @@ -0,0 +1,36 @@ +// +// Visage.cpp +// interface +// +// Created by Andrzej Kapolka on 2/11/14. +// Copyright (c) 2014 High Fidelity, Inc. All rights reserved. +// + +#ifdef HAVE_VISAGE +#include +#endif + +#include + +#include "Visage.h" + +namespace VisageSDK { + void initializeLicenseManager(char* licenseKeyFileName); +} + +using namespace VisageSDK; + +Visage::Visage() { +#ifdef HAVE_VISAGE + switchToResourcesParentIfRequired(); + initializeLicenseManager("resources/visage/license.vlc"); + _tracker = new VisageTracker2("resources/visage/Facial Features Tracker - Asymmetric.cfg"); + _tracker->trackFromCam(); +#endif +} + +Visage::~Visage() { +#ifdef HAVE_VISAGE + delete _tracker; +#endif +} diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h new file mode 100644 index 0000000000..1b9d283bb3 --- /dev/null +++ b/interface/src/devices/Visage.h @@ -0,0 +1,28 @@ +// +// Visage.h +// interface +// +// Created by Andrzej Kapolka on 2/11/14. +// Copyright (c) 2014 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__Visage__ +#define __interface__Visage__ + +namespace VisageSDK { + class VisageTracker2; +} + +/// Handles input from the Visage webcam feature tracking software. +class Visage { +public: + + Visage(); + ~Visage(); + +private: + + VisageSDK::VisageTracker2* _tracker; +}; + +#endif /* defined(__interface__Visage__) */ From b3fba320fcd3dced1f9252637eff016a89b23482 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 11 Feb 2014 17:03:24 -0800 Subject: [PATCH 03/21] We need the dependencies, too. --- cmake/modules/FindVisage.cmake | 32 ++++++++++++++++++++++++---- interface/external/visage/readme.txt | 2 +- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/cmake/modules/FindVisage.cmake b/cmake/modules/FindVisage.cmake index a8d1acc30b..600164908c 100644 --- a/cmake/modules/FindVisage.cmake +++ b/cmake/modules/FindVisage.cmake @@ -16,12 +16,36 @@ if (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) # in cache already set(VISAGE_FOUND TRUE) else (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) - find_path(VISAGE_INCLUDE_DIRS VisageTracker2.h ${VISAGE_ROOT_DIR}/include) - + find_path(VISAGE_INCLUDE_DIR VisageTracker2.h ${VISAGE_ROOT_DIR}/include) + if (APPLE) - find_library(VISAGE_LIBRARIES libvscore.a ${VISAGE_ROOT_DIR}/lib) + find_path(VISAGE_OPENCV_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/include) + if (VISAGE_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR) + set(VISAGE_INCLUDE_DIRS "${VISAGE_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR}" CACHE INTERNAL "Visage include dirs") + endif (VISAGE_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR) + + find_library(VISAGE_CORE_LIBRARY libvscore.a ${VISAGE_ROOT_DIR}/lib) + find_library(VISAGE_VISION_LIBRARY libvsvision.a ${VISAGE_ROOT_DIR}/lib) + find_library(VISAGE_OPENCV_LIBRARY libOpenCV.a ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/lib) + if (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) + set(VISAGE_LIBRARIES "${VISAGE_CORE_LIBRARY};${VISAGE_VISION_LIBRARY};${VISAGE_OPENCV_LIBRARY}" + CACHE INTERNAL "Visage libraries") + endif (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) + elseif (WIN32) - find_library(VISAGE_LIBRARIES vscore.dll ${VISAGE_ROOT_DIR}/lib) + find_path(VISAGE_OPENCV_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV_Win32/include) + if (VISAGE_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR) + set(VISAGE_INCLUDE_DIRS "${VISAGE_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR}" CACHE INTERNAL "Visage include dirs") + endif (VISAGE_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR) + + find_library(VISAGE_CORE_LIBRARY vscore.lib ${VISAGE_ROOT_DIR}/lib) + find_library(VISAGE_VISION_LIBRARY vsvision.lib ${VISAGE_ROOT_DIR}/lib) + find_library(VISAGE_OPENCV_LIBRARY OpenCV.lib ${VISAGE_ROOT_DIR}/dependencies/OpenCV_Win32/lib) + if (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) + set(VISAGE_LIBRARIES "${VISAGE_CORE_LIBRARY};${VISAGE_VISION_LIBRARY};${VISAGE_OPENCV_LIBRARY}" + CACHE INTERNAL "Visage libraries") + endif (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) + endif () if (VISAGE_INCLUDE_DIRS AND VISAGE_LIBRARIES) diff --git a/interface/external/visage/readme.txt b/interface/external/visage/readme.txt index dfb3da3969..8aa28f81c4 100644 --- a/interface/external/visage/readme.txt +++ b/interface/external/visage/readme.txt @@ -2,7 +2,7 @@ Instructions for adding the Visage driver to Interface Andrzej Kapolka, February 11, 2014 -1. Copy the Visage sdk folders (lib, include) into the interface/external/visage folder. +1. Copy the Visage sdk folders (lib, include, dependencies) into the interface/external/visage folder. This readme.txt should be there as well. 2. Copy the Visage configuration data folder (visageSDK-MacOS/Samples/MacOSX/data/) to interface/resources/visage From a9b415b0f9dc309677109cf792ede2e6a81d1a01 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 11 Feb 2014 17:22:40 -0800 Subject: [PATCH 04/21] More build settings. --- cmake/modules/FindVisage.cmake | 10 +++++++--- interface/CMakeLists.txt | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cmake/modules/FindVisage.cmake b/cmake/modules/FindVisage.cmake index 600164908c..0b5ac5f563 100644 --- a/cmake/modules/FindVisage.cmake +++ b/cmake/modules/FindVisage.cmake @@ -19,10 +19,14 @@ else (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) find_path(VISAGE_INCLUDE_DIR VisageTracker2.h ${VISAGE_ROOT_DIR}/include) if (APPLE) + find_path(VISAGE_XML_INCLUDE_DIR libxml/xmlreader.h /usr/include/libxml2) find_path(VISAGE_OPENCV_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/include) - if (VISAGE_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR) - set(VISAGE_INCLUDE_DIRS "${VISAGE_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR}" CACHE INTERNAL "Visage include dirs") - endif (VISAGE_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR) + find_path(VISAGE_OPENCV2_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/include/opencv2) + if (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) + set(VISAGE_INCLUDE_DIRS + "${VISAGE_INCLUDE_DIR};${VISAGE_XML_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR};${VISAGE_OPENCV2_INCLUDE_DIR}" + CACHE INTERNAL "Visage include dirs") + endif (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) find_library(VISAGE_CORE_LIBRARY libvscore.a ${VISAGE_ROOT_DIR}/lib) find_library(VISAGE_VISION_LIBRARY libvsvision.a ${VISAGE_ROOT_DIR}/lib) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 188e7a6995..81170b0a57 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -148,10 +148,11 @@ endif (SIXENSE_FOUND AND NOT DISABLE_SIXENSE) # likewise with Visage library for webcam feature tracking if (VISAGE_FOUND AND NOT DISABLE_VISAGE) - add_definitions(-DHAVE_VISAGE) + add_definitions(-DHAVE_VISAGE -DVISAGE_STATIC) include_directories(SYSTEM ${VISAGE_INCLUDE_DIRS}) if (APPLE OR UNIX) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${VISAGE_INCLUDE_DIRS}") + add_definitions(_DMAC_OS_X) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-comment -isystem ${VISAGE_INCLUDE_DIRS}") endif (APPLE OR UNIX) target_link_libraries(${TARGET_NAME} ${VISAGE_LIBRARIES}) endif (VISAGE_FOUND AND NOT DISABLE_VISAGE) From 87f7e2465fd5bbbdd9fa399627c5d171e547607f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 11 Feb 2014 17:25:25 -0800 Subject: [PATCH 05/21] Missed a spot. --- cmake/modules/FindVisage.cmake | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmake/modules/FindVisage.cmake b/cmake/modules/FindVisage.cmake index 0b5ac5f563..4c600755cd 100644 --- a/cmake/modules/FindVisage.cmake +++ b/cmake/modules/FindVisage.cmake @@ -21,7 +21,7 @@ else (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) if (APPLE) find_path(VISAGE_XML_INCLUDE_DIR libxml/xmlreader.h /usr/include/libxml2) find_path(VISAGE_OPENCV_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/include) - find_path(VISAGE_OPENCV2_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/include/opencv2) + find_path(VISAGE_OPENCV2_INCLUDE_DIR opencv.hpp ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/include/opencv2) if (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) set(VISAGE_INCLUDE_DIRS "${VISAGE_INCLUDE_DIR};${VISAGE_XML_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR};${VISAGE_OPENCV2_INCLUDE_DIR}" @@ -37,10 +37,14 @@ else (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) endif (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) elseif (WIN32) + find_path(VISAGE_XML_INCLUDE_DIR libxml/xmlreader.h /usr/include/libxml2) find_path(VISAGE_OPENCV_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV_Win32/include) - if (VISAGE_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR) - set(VISAGE_INCLUDE_DIRS "${VISAGE_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR}" CACHE INTERNAL "Visage include dirs") - endif (VISAGE_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR) + find_path(VISAGE_OPENCV2_INCLUDE_DIR opencv.hpp ${VISAGE_ROOT_DIR}/dependencies/OpenCV_Win32/include/opencv2) + if (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) + set(VISAGE_INCLUDE_DIRS + "${VISAGE_INCLUDE_DIR};${VISAGE_XML_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR};${VISAGE_OPENCV2_INCLUDE_DIR}" + CACHE INTERNAL "Visage include dirs") + endif (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) find_library(VISAGE_CORE_LIBRARY vscore.lib ${VISAGE_ROOT_DIR}/lib) find_library(VISAGE_VISION_LIBRARY vsvision.lib ${VISAGE_ROOT_DIR}/lib) From c07801261415bee79711ddb97ba4ef69b2fb245b Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 11 Feb 2014 17:26:31 -0800 Subject: [PATCH 06/21] Sniffing glue, wrong day, etc. --- interface/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 81170b0a57..e000efa695 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -151,7 +151,7 @@ if (VISAGE_FOUND AND NOT DISABLE_VISAGE) add_definitions(-DHAVE_VISAGE -DVISAGE_STATIC) include_directories(SYSTEM ${VISAGE_INCLUDE_DIRS}) if (APPLE OR UNIX) - add_definitions(_DMAC_OS_X) + add_definitions(-DMAC_OS_X) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-comment -isystem ${VISAGE_INCLUDE_DIRS}") endif (APPLE OR UNIX) target_link_libraries(${TARGET_NAME} ${VISAGE_LIBRARIES}) From 8206f8d2b3ef4f972aaa8abc8732dda697678ea2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 11 Feb 2014 19:00:41 -0800 Subject: [PATCH 07/21] Fixes for Windows. Compiles, but the license doesn't work yet. --- cmake/modules/FindVisage.cmake | 8 ++++---- interface/src/devices/Visage.cpp | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cmake/modules/FindVisage.cmake b/cmake/modules/FindVisage.cmake index 4c600755cd..84f672525d 100644 --- a/cmake/modules/FindVisage.cmake +++ b/cmake/modules/FindVisage.cmake @@ -37,9 +37,9 @@ else (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) endif (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) elseif (WIN32) - find_path(VISAGE_XML_INCLUDE_DIR libxml/xmlreader.h /usr/include/libxml2) - find_path(VISAGE_OPENCV_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV_Win32/include) - find_path(VISAGE_OPENCV2_INCLUDE_DIR opencv.hpp ${VISAGE_ROOT_DIR}/dependencies/OpenCV_Win32/include/opencv2) + find_path(VISAGE_XML_INCLUDE_DIR libxml/xmlreader.h ${VISAGE_ROOT_DIR}/dependencies/libxml2/include) + find_path(VISAGE_OPENCV_INCLUDE_DIR opencv/cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV/include) + find_path(VISAGE_OPENCV2_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV/include/opencv) if (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) set(VISAGE_INCLUDE_DIRS "${VISAGE_INCLUDE_DIR};${VISAGE_XML_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR};${VISAGE_OPENCV2_INCLUDE_DIR}" @@ -48,7 +48,7 @@ else (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) find_library(VISAGE_CORE_LIBRARY vscore.lib ${VISAGE_ROOT_DIR}/lib) find_library(VISAGE_VISION_LIBRARY vsvision.lib ${VISAGE_ROOT_DIR}/lib) - find_library(VISAGE_OPENCV_LIBRARY OpenCV.lib ${VISAGE_ROOT_DIR}/dependencies/OpenCV_Win32/lib) + find_library(VISAGE_OPENCV_LIBRARY opencv_core243.lib ${VISAGE_ROOT_DIR}/dependencies/OpenCV/lib) if (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) set(VISAGE_LIBRARIES "${VISAGE_CORE_LIBRARY};${VISAGE_VISION_LIBRARY};${VISAGE_OPENCV_LIBRARY}" CACHE INTERNAL "Visage libraries") diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index 4b986d7db4..f69121bf7f 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -6,16 +6,20 @@ // Copyright (c) 2014 High Fidelity, Inc. All rights reserved. // +#include + #ifdef HAVE_VISAGE #include #endif -#include - #include "Visage.h" namespace VisageSDK { - void initializeLicenseManager(char* licenseKeyFileName); +#ifdef WIN32 + void __declspec(dllimport) initializeLicenseManager(char* licenseKeyFileName); +#else + void initializeLicenseManager(char* licenseKeyFileName); +#endif } using namespace VisageSDK; @@ -23,7 +27,7 @@ using namespace VisageSDK; Visage::Visage() { #ifdef HAVE_VISAGE switchToResourcesParentIfRequired(); - initializeLicenseManager("resources/visage/license.vlc"); + initializeLicenseManager("resources/visage"); _tracker = new VisageTracker2("resources/visage/Facial Features Tracker - Asymmetric.cfg"); _tracker->trackFromCam(); #endif From 861ae12151c93f69293a49e50d5eea098b378154 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 13 Feb 2014 16:53:17 -0800 Subject: [PATCH 08/21] Added Visage stuff to .gitignore. --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 62686287bc..087e24208d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,11 @@ interface/external/Leap/util/ interface/external/Sixense/include/ interface/external/Sixense/lib/ +# Ignore Visage +interface/external/visage/dependencies/ +interface/external/visage/include/ +interface/external/visage/lib/ +interface/resources/visage/ + # Ignore interfaceCache for Linux users interface/interfaceCache/ From 8be17bf02aa3e76cc576baca011a2a160f240b07 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 14:27:06 -0800 Subject: [PATCH 09/21] Attempting to get Visage building in on OS X. --- interface/CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index c3f7ebc286..13238d5c56 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -156,10 +156,13 @@ endif (SIXENSE_FOUND AND NOT DISABLE_SIXENSE) if (VISAGE_FOUND AND NOT DISABLE_VISAGE) add_definitions(-DHAVE_VISAGE -DVISAGE_STATIC) include_directories(SYSTEM ${VISAGE_INCLUDE_DIRS}) - if (APPLE OR UNIX) + if (APPLE) add_definitions(-DMAC_OS_X) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-comment -isystem ${VISAGE_INCLUDE_DIRS}") - endif (APPLE OR UNIX) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-comment -isystem -std=c++0x ${VISAGE_INCLUDE_DIRS}") + find_library(AVFoundation AVFoundation) + find_library(CoreMedia CoreMedia) + target_link_libraries(${TARGET_NAME} ${AVFoundation} ${CoreMedia}) + endif (APPLE) target_link_libraries(${TARGET_NAME} ${VISAGE_LIBRARIES}) endif (VISAGE_FOUND AND NOT DISABLE_VISAGE) From a89959cd4768799c423e9a1b714d6d9884743c1f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 15:09:34 -0800 Subject: [PATCH 10/21] Get rid of this line. --- interface/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 13238d5c56..0c94be59f3 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -158,7 +158,7 @@ if (VISAGE_FOUND AND NOT DISABLE_VISAGE) include_directories(SYSTEM ${VISAGE_INCLUDE_DIRS}) if (APPLE) add_definitions(-DMAC_OS_X) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-comment -isystem -std=c++0x ${VISAGE_INCLUDE_DIRS}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-comment -isystem ${VISAGE_INCLUDE_DIRS}") find_library(AVFoundation AVFoundation) find_library(CoreMedia CoreMedia) target_link_libraries(${TARGET_NAME} ${AVFoundation} ${CoreMedia}) From 0547fa61b4b18218400e222b5c491c893ed7d7db Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 15:17:16 -0800 Subject: [PATCH 11/21] Link in the new standard library, which Visage uses. --- interface/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 0c94be59f3..a0a9033187 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -161,7 +161,8 @@ if (VISAGE_FOUND AND NOT DISABLE_VISAGE) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-comment -isystem ${VISAGE_INCLUDE_DIRS}") find_library(AVFoundation AVFoundation) find_library(CoreMedia CoreMedia) - target_link_libraries(${TARGET_NAME} ${AVFoundation} ${CoreMedia}) + find_library(NEW_STD_LIBRARY libc++.dylib /usr/lib/) + target_link_libraries(${TARGET_NAME} ${AVFoundation} ${CoreMedia} ${NEW_STD_LIBRARY}) endif (APPLE) target_link_libraries(${TARGET_NAME} ${VISAGE_LIBRARIES}) endif (VISAGE_FOUND AND NOT DISABLE_VISAGE) From cc6b663ea6619f98e228272be573583f0348e12c Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 15:37:40 -0800 Subject: [PATCH 12/21] Use the ABI-compatible library (hopefully). --- interface/CMakeLists.txt | 2 +- interface/src/devices/Visage.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index a0a9033187..32e75c9f34 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -161,7 +161,7 @@ if (VISAGE_FOUND AND NOT DISABLE_VISAGE) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-comment -isystem ${VISAGE_INCLUDE_DIRS}") find_library(AVFoundation AVFoundation) find_library(CoreMedia CoreMedia) - find_library(NEW_STD_LIBRARY libc++.dylib /usr/lib/) + find_library(NEW_STD_LIBRARY libc++abi.dylib /usr/lib/) target_link_libraries(${TARGET_NAME} ${AVFoundation} ${CoreMedia} ${NEW_STD_LIBRARY}) endif (APPLE) target_link_libraries(${TARGET_NAME} ${VISAGE_LIBRARIES}) diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index f69121bf7f..6a612eb59d 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -27,7 +27,7 @@ using namespace VisageSDK; Visage::Visage() { #ifdef HAVE_VISAGE switchToResourcesParentIfRequired(); - initializeLicenseManager("resources/visage"); + initializeLicenseManager("resources/visage/license.vlc"); _tracker = new VisageTracker2("resources/visage/Facial Features Tracker - Asymmetric.cfg"); _tracker->trackFromCam(); #endif @@ -35,6 +35,7 @@ Visage::Visage() { Visage::~Visage() { #ifdef HAVE_VISAGE + _tracker->stop(); delete _tracker; #endif } From f83254882aff25e823a1d19c64962616736c1492 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 16:07:08 -0800 Subject: [PATCH 13/21] Attempting to get the head rotation from Visage. --- interface/src/Application.cpp | 10 ++++++++++ interface/src/Application.h | 2 ++ interface/src/avatar/MyAvatar.cpp | 24 +++++++++++++++++------- interface/src/devices/Visage.cpp | 28 ++++++++++++++++++++++++---- interface/src/devices/Visage.h | 16 ++++++++++++++++ 5 files changed, 69 insertions(+), 11 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index b316548ad4..098b42dccf 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2041,6 +2041,15 @@ void Application::updateFaceshift() { } } +void Application::updateVisage() { + + bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); + PerformanceWarning warn(showWarnings, "Application::updateVisage()"); + + // Update Visage + _visage.update(); +} + void Application::updateMyAvatarLookAtPosition(glm::vec3& lookAtSpot) { bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); @@ -2318,6 +2327,7 @@ void Application::update(float deltaTime) { glm::vec3 lookAtSpot; updateFaceshift(); + updateVisage(); _myAvatar->updateLookAtTargetAvatar(lookAtSpot); updateMyAvatarLookAtPosition(lookAtSpot); diff --git a/interface/src/Application.h b/interface/src/Application.h index f9c1005c13..06a19f13b1 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -161,6 +161,7 @@ public: const glm::vec3& getMouseRayOrigin() const { return _mouseRayOrigin; } const glm::vec3& getMouseRayDirection() const { return _mouseRayDirection; } Faceshift* getFaceshift() { return &_faceshift; } + Visage* getVisage() { return &_visage; } SixenseManager* getSixenseManager() { return &_sixenseManager; } BandwidthMeter* getBandwidthMeter() { return &_bandwidthMeter; } QSettings* getSettings() { return _settings; } @@ -284,6 +285,7 @@ private: // Various helper functions called during update() void updateMouseRay(); void updateFaceshift(); + void updateVisage(); void updateMyAvatarLookAtPosition(glm::vec3& lookAtSpot); void updateHoverVoxels(float deltaTime, float& distance, BoxFace& face); void updateMouseVoxels(float deltaTime, float& distance, BoxFace& face); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 17b80089a7..51de5a944c 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -337,22 +337,32 @@ const float MAX_PITCH = 90.0f; // Update avatar head rotation with sensor data void MyAvatar::updateFromGyros(float deltaTime) { Faceshift* faceshift = Application::getInstance()->getFaceshift(); + Visage* visage = Application::getInstance()->getVisage(); glm::vec3 estimatedPosition, estimatedRotation; + bool trackerActive = false; if (faceshift->isActive()) { estimatedPosition = faceshift->getHeadTranslation(); estimatedRotation = safeEulerAngles(faceshift->getHeadRotation()); + trackerActive = true; + + } else if (visage->isActive()) { + estimatedPosition = visage->getHeadTranslation(); + estimatedRotation = safeEulerAngles(visage->getHeadRotation()); + trackerActive = true; + } + if (trackerActive) { // Rotate the body if the head is turned beyond the screen if (Menu::getInstance()->isOptionChecked(MenuOption::TurnWithHead)) { - const float FACESHIFT_YAW_TURN_SENSITIVITY = 0.5f; - const float FACESHIFT_MIN_YAW_TURN = 15.f; - const float FACESHIFT_MAX_YAW_TURN = 50.f; - if ( (fabs(estimatedRotation.y) > FACESHIFT_MIN_YAW_TURN) && - (fabs(estimatedRotation.y) < FACESHIFT_MAX_YAW_TURN) ) { + const float TRACKER_YAW_TURN_SENSITIVITY = 0.5f; + const float TRACKER_MIN_YAW_TURN = 15.f; + const float TRACKER_MAX_YAW_TURN = 50.f; + if ( (fabs(estimatedRotation.y) > TRACKER_MIN_YAW_TURN) && + (fabs(estimatedRotation.y) < TRACKER_MAX_YAW_TURN) ) { if (estimatedRotation.y > 0.f) { - _bodyYawDelta += (estimatedRotation.y - FACESHIFT_MIN_YAW_TURN) * FACESHIFT_YAW_TURN_SENSITIVITY; + _bodyYawDelta += (estimatedRotation.y - TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; } else { - _bodyYawDelta += (estimatedRotation.y + FACESHIFT_MIN_YAW_TURN) * FACESHIFT_YAW_TURN_SENSITIVITY; + _bodyYawDelta += (estimatedRotation.y + TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; } } } diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index 6a612eb59d..59b3ac06f8 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -24,18 +24,38 @@ namespace VisageSDK { using namespace VisageSDK; -Visage::Visage() { +Visage::Visage() : + _active(false) { #ifdef HAVE_VISAGE switchToResourcesParentIfRequired(); initializeLicenseManager("resources/visage/license.vlc"); _tracker = new VisageTracker2("resources/visage/Facial Features Tracker - Asymmetric.cfg"); - _tracker->trackFromCam(); + if (_tracker->trackFromCam()) { + _data = new FaceData(); + + } else { + delete _tracker; + _tracker = NULL; + } #endif } Visage::~Visage() { #ifdef HAVE_VISAGE - _tracker->stop(); - delete _tracker; + if (_tracker) { + _tracker->stop(); + delete _tracker; + delete _data; + } +#endif +} + +void Visage::update() { +#ifdef HAVE_VISAGE + _active = (_tracker->getTrackingData(_data) == TRACK_STAT_OK); + if (!_active) { + return; + } + _headRotation = glm::quat(glm::vec3(_data->faceRotation[0], _data->faceRotation[1], _data->faceRotation[2])); #endif } diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h index 1b9d283bb3..f16119cdbd 100644 --- a/interface/src/devices/Visage.h +++ b/interface/src/devices/Visage.h @@ -9,7 +9,11 @@ #ifndef __interface__Visage__ #define __interface__Visage__ +#include +#include + namespace VisageSDK { + class FaceData; class VisageTracker2; } @@ -20,9 +24,21 @@ public: Visage(); ~Visage(); + bool isActive() const { return _active; } + + const glm::quat& getHeadRotation() const { return _headRotation; } + const glm::vec3& getHeadTranslation() const { return _headTranslation; } + + void update(); + private: VisageSDK::VisageTracker2* _tracker; + VisageSDK::FaceData* _data; + + bool _active; + glm::quat _headRotation; + glm::vec3 _headTranslation; }; #endif /* defined(__interface__Visage__) */ From ada2594ad17462ef92c625877a544be0594fe517 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 16:36:54 -0800 Subject: [PATCH 14/21] Working on getting the head translation, too. --- interface/src/Application.cpp | 1 + interface/src/devices/Visage.cpp | 17 +++++++++++++---- interface/src/devices/Visage.h | 5 ++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 098b42dccf..fc8bf8e899 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3831,6 +3831,7 @@ void Application::resetSensors() { _mouseY = _glWidget->height() / 2; _faceshift.reset(); + _visage.reset(); if (OculusManager::isConnected()) { OculusManager::reset(); diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index 59b3ac06f8..94992417f4 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -24,11 +24,15 @@ namespace VisageSDK { using namespace VisageSDK; +const glm::vec3 DEFAULT_HEAD_ORIGIN(0.0f, 0.0f, 0.3f); + Visage::Visage() : - _active(false) { + _active(false), + _headOrigin(DEFAULT_HEAD_ORIGIN) { #ifdef HAVE_VISAGE switchToResourcesParentIfRequired(); - initializeLicenseManager("resources/visage/license.vlc"); + QByteArray licensePath = "resources/visage/license.vlc"; + initializeLicenseManager(licensePath); _tracker = new VisageTracker2("resources/visage/Facial Features Tracker - Asymmetric.cfg"); if (_tracker->trackFromCam()) { _data = new FaceData(); @@ -52,10 +56,15 @@ Visage::~Visage() { void Visage::update() { #ifdef HAVE_VISAGE - _active = (_tracker->getTrackingData(_data) == TRACK_STAT_OK); + _active = (_tracker && _tracker->getTrackingData(_data) == TRACK_STAT_OK); if (!_active) { return; } - _headRotation = glm::quat(glm::vec3(_data->faceRotation[0], _data->faceRotation[1], _data->faceRotation[2])); + _headRotation = glm::quat(glm::vec3(-_data->faceRotation[0], -_data->faceRotation[1], _data->faceRotation[2])); + _headTranslation = glm::vec3(_data->faceTranslation[0], _data->faceTranslation[1], _data->faceTranslation[2]) - _headOrigin; #endif } + +void Visage::reset() { + _headOrigin += _headTranslation; +} diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h index f16119cdbd..20c361bda0 100644 --- a/interface/src/devices/Visage.h +++ b/interface/src/devices/Visage.h @@ -13,8 +13,8 @@ #include namespace VisageSDK { - class FaceData; class VisageTracker2; + struct FaceData; } /// Handles input from the Visage webcam feature tracking software. @@ -30,6 +30,7 @@ public: const glm::vec3& getHeadTranslation() const { return _headTranslation; } void update(); + void reset(); private: @@ -39,6 +40,8 @@ private: bool _active; glm::quat _headRotation; glm::vec3 _headTranslation; + + glm::vec3 _headOrigin; }; #endif /* defined(__interface__Visage__) */ From fe7970665389307d22f6c285f56de89b0e7b55d0 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 16:42:30 -0800 Subject: [PATCH 15/21] Working on the translation scale. --- interface/src/devices/Visage.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index 94992417f4..cce5f66340 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -32,7 +32,7 @@ Visage::Visage() : #ifdef HAVE_VISAGE switchToResourcesParentIfRequired(); QByteArray licensePath = "resources/visage/license.vlc"; - initializeLicenseManager(licensePath); + initializeLicenseManager(licensePath.data()); _tracker = new VisageTracker2("resources/visage/Facial Features Tracker - Asymmetric.cfg"); if (_tracker->trackFromCam()) { _data = new FaceData(); @@ -54,17 +54,20 @@ Visage::~Visage() { #endif } +const float TRANSLATION_SCALE = 50.0f; + void Visage::update() { #ifdef HAVE_VISAGE _active = (_tracker && _tracker->getTrackingData(_data) == TRACK_STAT_OK); if (!_active) { return; } - _headRotation = glm::quat(glm::vec3(-_data->faceRotation[0], -_data->faceRotation[1], _data->faceRotation[2])); - _headTranslation = glm::vec3(_data->faceTranslation[0], _data->faceTranslation[1], _data->faceTranslation[2]) - _headOrigin; + _headRotation = glm::quat(glm::vec3(-_data->faceRotation[0], -_data->faceRotation[1], _data->faceRotation[2])); + _headTranslation = (glm::vec3(_data->faceTranslation[0], _data->faceTranslation[1], _data->faceTranslation[2]) - + _headOrigin) * TRANSLATION_SCALE; #endif } void Visage::reset() { - _headOrigin += _headTranslation; + _headOrigin += _headTranslation / TRANSLATION_SCALE; } From 9c4143e1beb7234ec03852b5001e850739f257e2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 17:03:43 -0800 Subject: [PATCH 16/21] Working on gaze deflection. --- interface/src/Application.cpp | 14 +++++++++++++- interface/src/devices/Visage.cpp | 10 +++++++--- interface/src/devices/Visage.h | 6 ++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index fc8bf8e899..88c7af102b 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2070,13 +2070,25 @@ void Application::updateMyAvatarLookAtPosition(glm::vec3& lookAtSpot) { } lookAtSpot = _mouseRayOrigin + _mouseRayDirection * distance; } + bool trackerActive = false; + float eyePitch, eyeYaw; if (_faceshift.isActive()) { + eyePitch = _faceshift.getEstimatedEyePitch(); + eyeYaw = _faceshift.getEstimatedEyeYaw(); + trackerActive = true; + + } else if (_visage.isActive()) { + eyePitch = _visage.getEstimatedEyePitch(); + eyeYaw = _visage.getEstimatedEyeYaw(); + trackerActive = true; + } + if (trackerActive) { // deflect using Faceshift gaze data glm::vec3 origin = _myAvatar->getHead().calculateAverageEyePosition(); float pitchSign = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? -1.0f : 1.0f; float deflection = Menu::getInstance()->getFaceshiftEyeDeflection(); lookAtSpot = origin + _myCamera.getRotation() * glm::quat(glm::radians(glm::vec3( - _faceshift.getEstimatedEyePitch() * pitchSign * deflection, _faceshift.getEstimatedEyeYaw() * deflection, 0.0f))) * + eyePitch * pitchSign * deflection, eyeYaw * deflection, 0.0f))) * glm::inverse(_myCamera.getRotation()) * (lookAtSpot - origin); } _myAvatar->getHead().setLookAtPosition(lookAtSpot); diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index cce5f66340..8696db3af4 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -24,11 +24,13 @@ namespace VisageSDK { using namespace VisageSDK; -const glm::vec3 DEFAULT_HEAD_ORIGIN(0.0f, 0.0f, 0.3f); +const glm::vec3 DEFAULT_HEAD_ORIGIN(0.0f, 0.0f, 0.7f); Visage::Visage() : _active(false), - _headOrigin(DEFAULT_HEAD_ORIGIN) { + _headOrigin(DEFAULT_HEAD_ORIGIN), + _estimatedEyePitch(0.0f), + _estimatedEyeYaw(0.0f) { #ifdef HAVE_VISAGE switchToResourcesParentIfRequired(); QByteArray licensePath = "resources/visage/license.vlc"; @@ -54,7 +56,7 @@ Visage::~Visage() { #endif } -const float TRANSLATION_SCALE = 50.0f; +const float TRANSLATION_SCALE = 20.0f; void Visage::update() { #ifdef HAVE_VISAGE @@ -65,6 +67,8 @@ void Visage::update() { _headRotation = glm::quat(glm::vec3(-_data->faceRotation[0], -_data->faceRotation[1], _data->faceRotation[2])); _headTranslation = (glm::vec3(_data->faceTranslation[0], _data->faceTranslation[1], _data->faceTranslation[2]) - _headOrigin) * TRANSLATION_SCALE; + _estimatedEyePitch = glm::degrees(_data->gazeDirection[1]); + _estimatedEyeYaw = glm::degrees(_data->gazeDirection[0]); #endif } diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h index 20c361bda0..966c41335e 100644 --- a/interface/src/devices/Visage.h +++ b/interface/src/devices/Visage.h @@ -29,6 +29,9 @@ public: const glm::quat& getHeadRotation() const { return _headRotation; } const glm::vec3& getHeadTranslation() const { return _headTranslation; } + float getEstimatedEyePitch() const { return _estimatedEyePitch; } + float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } + void update(); void reset(); @@ -42,6 +45,9 @@ private: glm::vec3 _headTranslation; glm::vec3 _headOrigin; + + float _estimatedEyePitch; + float _estimatedEyeYaw; }; #endif /* defined(__interface__Visage__) */ From e97a912d9746600afd880cf74bab4d5b64e592ac Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 17:19:29 -0800 Subject: [PATCH 17/21] Working on blendshape coefficients. --- interface/src/avatar/Head.cpp | 26 +++++++++++--------------- interface/src/devices/Visage.cpp | 11 +++++++++-- interface/src/devices/Visage.h | 6 ++++++ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index ddb0660364..5c6100764a 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -62,24 +62,20 @@ void Head::simulate(float deltaTime, bool isMine) { // Update audio trailing average for rendering facial animations Faceshift* faceshift = Application::getInstance()->getFaceshift(); + Visage* visage = Application::getInstance()->getVisage(); if (isMine) { - _isFaceshiftConnected = faceshift->isActive(); + _isFaceshiftConnected = false; + if (faceshift->isActive()) { + _blendshapeCoefficients = faceshift->getBlendshapeCoefficients(); + _isFaceshiftConnected = true; + + } else if (visage->isActive()) { + _blendshapeCoefficients = visage->getBlendshapeCoefficients(); + _isFaceshiftConnected = true; + } } - if (isMine && faceshift->isActive()) { - const float EYE_OPEN_SCALE = 0.5f; - _leftEyeBlink = faceshift->getLeftBlink() - EYE_OPEN_SCALE * faceshift->getLeftEyeOpen(); - _rightEyeBlink = faceshift->getRightBlink() - EYE_OPEN_SCALE * faceshift->getRightEyeOpen(); - - // set these values based on how they'll be used. if we use faceshift in the long term, we'll want a complete - // mapping between their blendshape coefficients and our avatar features - const float MOUTH_SIZE_SCALE = 2500.0f; - _averageLoudness = faceshift->getMouthSize() * faceshift->getMouthSize() * MOUTH_SIZE_SCALE; - const float BROW_HEIGHT_SCALE = 0.005f; - _browAudioLift = faceshift->getBrowUpCenter() * BROW_HEIGHT_SCALE; - _blendshapeCoefficients = faceshift->getBlendshapeCoefficients(); - - } else if (!_isFaceshiftConnected) { + if (!_isFaceshiftConnected) { // Update eye saccades const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; const float AVERAGE_SACCADE_INTERVAL = 4.0f; diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index 8696db3af4..c6fb5b9668 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -67,8 +67,15 @@ void Visage::update() { _headRotation = glm::quat(glm::vec3(-_data->faceRotation[0], -_data->faceRotation[1], _data->faceRotation[2])); _headTranslation = (glm::vec3(_data->faceTranslation[0], _data->faceTranslation[1], _data->faceTranslation[2]) - _headOrigin) * TRANSLATION_SCALE; - _estimatedEyePitch = glm::degrees(_data->gazeDirection[1]); - _estimatedEyeYaw = glm::degrees(_data->gazeDirection[0]); + _estimatedEyePitch = glm::degrees(-_data->gazeDirection[1]); + _estimatedEyeYaw = glm::degrees(-_data->gazeDirection[0]); + + for (int i = 0; i < _data->actionUnitCount; i++) { + if (!_data->actionUnitsUsed[i]) { + continue; + } + qDebug() << _data->actionUnitsNames[i] << _data->actionUnits[i]; + } #endif } diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h index 966c41335e..383c81be54 100644 --- a/interface/src/devices/Visage.h +++ b/interface/src/devices/Visage.h @@ -9,6 +9,8 @@ #ifndef __interface__Visage__ #define __interface__Visage__ +#include + #include #include @@ -32,6 +34,8 @@ public: float getEstimatedEyePitch() const { return _estimatedEyePitch; } float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } + const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + void update(); void reset(); @@ -48,6 +52,8 @@ private: float _estimatedEyePitch; float _estimatedEyeYaw; + + std::vector _blendshapeCoefficients; }; #endif /* defined(__interface__Visage__) */ From 636e3b1910952a5ce0d1ff92c89554603cc24d7a Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 18:24:25 -0800 Subject: [PATCH 18/21] Working on blendshape integration. --- interface/src/devices/Visage.cpp | 56 +++++++++++++++++++++++++++++- interface/src/devices/Visage.h | 4 +++ interface/src/renderer/FBXReader.h | 3 ++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index c6fb5b9668..2ed36e9594 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -6,6 +6,8 @@ // Copyright (c) 2014 High Fidelity, Inc. All rights reserved. // +#include + #include #ifdef HAVE_VISAGE @@ -13,6 +15,7 @@ #endif #include "Visage.h" +#include "renderer/FBXReader.h" namespace VisageSDK { #ifdef WIN32 @@ -56,6 +59,43 @@ Visage::~Visage() { #endif } +static QHash createBlendshapeIndices() { + QHash blendshapeMap; + blendshapeMap.insert("Sneer", "au_nose_wrinkler"); + blendshapeMap.insert("JawFwd", "au_jaw_z_push"); + blendshapeMap.insert("JawLeft", "au_jaw_x_push"); + blendshapeMap.insert("JawOpen", "au_jaw_drop"); + blendshapeMap.insert("LipsLowerDown", "au_lower_lip_drop"); + blendshapeMap.insert("LipsUpperUp", "au_upper_lip_raiser"); + blendshapeMap.insert("LipsStretch_L", "au_lip_stretcher_left"); + blendshapeMap.insert("BrowsU_L", "au_left_outer_brow_raiser"); + blendshapeMap.insert("BrowsU_C", "au_left_inner_brow_raiser"); + blendshapeMap.insert("BrowsD_L", "au_left_brow_lowerer"); + blendshapeMap.insert("LipsStretch_R", "au_lip_stretcher_right"); + blendshapeMap.insert("BrowsU_R", "au_right_outer_brow_raiser"); + blendshapeMap.insert("BrowsU_C", "au_right_inner_brow_raiser"); + blendshapeMap.insert("BrowsD_R", "au_right_brow_lowerer"); + + QHash blendshapeIndices; + for (int i = 0;; i++) { + QByteArray blendshape = FACESHIFT_BLENDSHAPES[i]; + if (blendshape.isEmpty()) { + break; + } + QByteArray mapping = blendshapeMap.value(blendshape); + if (!mapping.isEmpty()) { + blendshapeIndices.insert(mapping, i + 1); + } + } + + return blendshapeIndices; +} + +static const QHash& getBlendshapeIndices() { + static QHash blendshapeIndices = createBlendshapeIndices(); + return blendshapeIndices; +} + const float TRANSLATION_SCALE = 20.0f; void Visage::update() { @@ -70,11 +110,25 @@ void Visage::update() { _estimatedEyePitch = glm::degrees(-_data->gazeDirection[1]); _estimatedEyeYaw = glm::degrees(-_data->gazeDirection[0]); + if (_blendshapeIndices.isEmpty()) { + _blendshapeIndices.resize(_data->actionUnitCount); + int maxIndex = -1; + for (int i = 0; i < _data->actionUnitCount; i++) { + int index = getBlendshapeIndices().value(_data->actionUnitsNames[i]) - 1; + maxIndex = qMax(maxIndex, _blendshapeIndices[i] = index); + } + _blendshapeCoefficients.resize(maxIndex + 1); + } + + qFill(_blendshapeCoefficients.begin(), _blendshapeCoefficients.end(), 0.0f); for (int i = 0; i < _data->actionUnitCount; i++) { if (!_data->actionUnitsUsed[i]) { continue; } - qDebug() << _data->actionUnitsNames[i] << _data->actionUnits[i]; + int index = _blendshapeIndices.at(i); + if (index != -1) { + _blendshapeCoefficients[index] = _data->actionUnits[i]; + } } #endif } diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h index 383c81be54..0559a859de 100644 --- a/interface/src/devices/Visage.h +++ b/interface/src/devices/Visage.h @@ -11,6 +11,8 @@ #include +#include + #include #include @@ -54,6 +56,8 @@ private: float _estimatedEyeYaw; std::vector _blendshapeCoefficients; + + QVector _blendshapeIndices; }; #endif /* defined(__interface__Visage__) */ diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 8efb23c98c..b89d0954b4 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -21,6 +21,9 @@ class FBXNode; typedef QList FBXNodeList; +/// The names of the blendshapes expected by Faceshift, terminated with an empty string. +extern const char* FACESHIFT_BLENDSHAPES[]; + /// A node within an FBX document. class FBXNode { public: From 8076f571fd5ea93aedfd91047e2e776458c5e3bc Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 18:39:38 -0800 Subject: [PATCH 19/21] Working on blinking, etc. --- interface/src/devices/Visage.cpp | 28 +++++++++++++++++++++++++++- interface/src/devices/Visage.h | 2 ++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index 2ed36e9594..8a6b9d5947 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -33,7 +33,10 @@ Visage::Visage() : _active(false), _headOrigin(DEFAULT_HEAD_ORIGIN), _estimatedEyePitch(0.0f), - _estimatedEyeYaw(0.0f) { + _estimatedEyeYaw(0.0f), + _leftInnerBrowIndex(0), + _rightInnerBrowIndex(0) { + #ifdef HAVE_VISAGE switchToResourcesParentIfRequired(); QByteArray licensePath = "resources/visage/license.vlc"; @@ -59,6 +62,10 @@ Visage::~Visage() { #endif } +static int leftEyeBlinkIndex = 0; +static int rightEyeBlinkIndex = 1; +static int centerBrowIndex = 16; + static QHash createBlendshapeIndices() { QHash blendshapeMap; blendshapeMap.insert("Sneer", "au_nose_wrinkler"); @@ -82,6 +89,15 @@ static QHash createBlendshapeIndices() { if (blendshape.isEmpty()) { break; } + if (blendshape == "EyeBlink_L") { + leftEyeBlinkIndex = i; + + } else if (blendshape == "EyeBlink_R") { + rightEyeBlinkIndex = i; + + } else if (blendshape == "BrowsU_C") { + centerBrowIndex = i; + } QByteArray mapping = blendshapeMap.value(blendshape); if (!mapping.isEmpty()) { blendshapeIndices.insert(mapping, i + 1); @@ -114,6 +130,12 @@ void Visage::update() { _blendshapeIndices.resize(_data->actionUnitCount); int maxIndex = -1; for (int i = 0; i < _data->actionUnitCount; i++) { + QByteArray name = _data->actionUnitsNames[i]; + if (name == "au_left_inner_brow_raiser") { + _leftInnerBrowIndex = i; + } else if (name == "au_right_inner_brow_raiser") { + _rightInnerBrowIndex = i; + } int index = getBlendshapeIndices().value(_data->actionUnitsNames[i]) - 1; maxIndex = qMax(maxIndex, _blendshapeIndices[i] = index); } @@ -130,6 +152,10 @@ void Visage::update() { _blendshapeCoefficients[index] = _data->actionUnits[i]; } } + _blendshapeCoefficients[leftEyeBlinkIndex] = 1.0f - _data->eyeClosure[0]; + _blendshapeCoefficients[rightEyeBlinkIndex] = 1.0f - _data->eyeClosure[1]; + _blendshapeCoefficients[centerBrowIndex] = (_data->actionUnits[_leftInnerBrowIndex] + + _data->actionUnits[_rightInnerBrowIndex]) * 0.5f; #endif } diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h index 0559a859de..124ef29e3b 100644 --- a/interface/src/devices/Visage.h +++ b/interface/src/devices/Visage.h @@ -58,6 +58,8 @@ private: std::vector _blendshapeCoefficients; QVector _blendshapeIndices; + int _leftInnerBrowIndex; + int _rightInnerBrowIndex; }; #endif /* defined(__interface__Visage__) */ From c4e643e3a4f6155c18eef4ee9e19c6017bfc129f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 18:43:45 -0800 Subject: [PATCH 20/21] Eyes were reversed. --- interface/src/devices/Visage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index 8a6b9d5947..b1a041c0dc 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -152,8 +152,8 @@ void Visage::update() { _blendshapeCoefficients[index] = _data->actionUnits[i]; } } - _blendshapeCoefficients[leftEyeBlinkIndex] = 1.0f - _data->eyeClosure[0]; - _blendshapeCoefficients[rightEyeBlinkIndex] = 1.0f - _data->eyeClosure[1]; + _blendshapeCoefficients[leftEyeBlinkIndex] = 1.0f - _data->eyeClosure[1]; + _blendshapeCoefficients[rightEyeBlinkIndex] = 1.0f - _data->eyeClosure[0]; _blendshapeCoefficients[centerBrowIndex] = (_data->actionUnits[_leftInnerBrowIndex] + _data->actionUnits[_rightInnerBrowIndex]) * 0.5f; #endif From a89983a1c0f28f5b8d8d8cb6bb273e8273e7e459 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Feb 2014 18:49:34 -0800 Subject: [PATCH 21/21] Removed tab, variable fix. --- interface/src/devices/Visage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp index b1a041c0dc..26e7993ecb 100644 --- a/interface/src/devices/Visage.cpp +++ b/interface/src/devices/Visage.cpp @@ -21,7 +21,7 @@ namespace VisageSDK { #ifdef WIN32 void __declspec(dllimport) initializeLicenseManager(char* licenseKeyFileName); #else - void initializeLicenseManager(char* licenseKeyFileName); + void initializeLicenseManager(char* licenseKeyFileName); #endif } @@ -136,7 +136,7 @@ void Visage::update() { } else if (name == "au_right_inner_brow_raiser") { _rightInnerBrowIndex = i; } - int index = getBlendshapeIndices().value(_data->actionUnitsNames[i]) - 1; + int index = getBlendshapeIndices().value(name) - 1; maxIndex = qMax(maxIndex, _blendshapeIndices[i] = index); } _blendshapeCoefficients.resize(maxIndex + 1);