diff --git a/.gitignore b/.gitignore index 62686287bc..087e24208d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,11 @@ interface/external/Leap/util/ interface/external/Sixense/include/ interface/external/Sixense/lib/ +# Ignore Visage +interface/external/visage/dependencies/ +interface/external/visage/include/ +interface/external/visage/lib/ +interface/resources/visage/ + # Ignore interfaceCache for Linux users interface/interfaceCache/ diff --git a/cmake/modules/FindVisage.cmake b/cmake/modules/FindVisage.cmake new file mode 100644 index 0000000000..84f672525d --- /dev/null +++ b/cmake/modules/FindVisage.cmake @@ -0,0 +1,76 @@ +# Try to find the Visage controller library +# +# You must provide a VISAGE_ROOT_DIR which contains lib and include directories +# +# Once done this will define +# +# VISAGE_FOUND - system found Visage +# VISAGE_INCLUDE_DIRS - the Visage include directory +# VISAGE_LIBRARIES - Link this to use Visage +# +# Created on 2/11/2014 by Andrzej Kapolka +# Copyright (c) 2014 High Fidelity +# + +if (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) + # in cache already + set(VISAGE_FOUND TRUE) +else (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) + find_path(VISAGE_INCLUDE_DIR VisageTracker2.h ${VISAGE_ROOT_DIR}/include) + + if (APPLE) + find_path(VISAGE_XML_INCLUDE_DIR libxml/xmlreader.h /usr/include/libxml2) + find_path(VISAGE_OPENCV_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/include) + find_path(VISAGE_OPENCV2_INCLUDE_DIR opencv.hpp ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/include/opencv2) + if (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) + set(VISAGE_INCLUDE_DIRS + "${VISAGE_INCLUDE_DIR};${VISAGE_XML_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR};${VISAGE_OPENCV2_INCLUDE_DIR}" + CACHE INTERNAL "Visage include dirs") + endif (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) + + find_library(VISAGE_CORE_LIBRARY libvscore.a ${VISAGE_ROOT_DIR}/lib) + find_library(VISAGE_VISION_LIBRARY libvsvision.a ${VISAGE_ROOT_DIR}/lib) + find_library(VISAGE_OPENCV_LIBRARY libOpenCV.a ${VISAGE_ROOT_DIR}/dependencies/OpenCV_MacOSX/lib) + if (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) + set(VISAGE_LIBRARIES "${VISAGE_CORE_LIBRARY};${VISAGE_VISION_LIBRARY};${VISAGE_OPENCV_LIBRARY}" + CACHE INTERNAL "Visage libraries") + endif (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) + + elseif (WIN32) + find_path(VISAGE_XML_INCLUDE_DIR libxml/xmlreader.h ${VISAGE_ROOT_DIR}/dependencies/libxml2/include) + find_path(VISAGE_OPENCV_INCLUDE_DIR opencv/cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV/include) + find_path(VISAGE_OPENCV2_INCLUDE_DIR cv.h ${VISAGE_ROOT_DIR}/dependencies/OpenCV/include/opencv) + if (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) + set(VISAGE_INCLUDE_DIRS + "${VISAGE_INCLUDE_DIR};${VISAGE_XML_INCLUDE_DIR};${VISAGE_OPENCV_INCLUDE_DIR};${VISAGE_OPENCV2_INCLUDE_DIR}" + CACHE INTERNAL "Visage include dirs") + endif (VISAGE_INCLUDE_DIR AND VISAGE_XML_INCLUDE_DIR AND VISAGE_OPENCV_INCLUDE_DIR AND VISAGE_OPENCV2_INCLUDE_DIR) + + find_library(VISAGE_CORE_LIBRARY vscore.lib ${VISAGE_ROOT_DIR}/lib) + find_library(VISAGE_VISION_LIBRARY vsvision.lib ${VISAGE_ROOT_DIR}/lib) + find_library(VISAGE_OPENCV_LIBRARY opencv_core243.lib ${VISAGE_ROOT_DIR}/dependencies/OpenCV/lib) + if (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) + set(VISAGE_LIBRARIES "${VISAGE_CORE_LIBRARY};${VISAGE_VISION_LIBRARY};${VISAGE_OPENCV_LIBRARY}" + CACHE INTERNAL "Visage libraries") + endif (VISAGE_CORE_LIBRARY AND VISAGE_VISION_LIBRARY AND VISAGE_OPENCV_LIBRARY) + + endif () + + if (VISAGE_INCLUDE_DIRS AND VISAGE_LIBRARIES) + set(VISAGE_FOUND TRUE) + endif (VISAGE_INCLUDE_DIRS AND VISAGE_LIBRARIES) + + if (VISAGE_FOUND) + if (NOT VISAGE_FIND_QUIETLY) + message(STATUS "Found Visage: ${VISAGE_LIBRARIES}") + endif (NOT VISAGE_FIND_QUIETLY) + else (VISAGE_FOUND) + if (VISAGE_FIND_REQUIRED) + message(FATAL_ERROR "Could not find Visage") + endif (VISAGE_FIND_REQUIRED) + endif (VISAGE_FOUND) + + # show the VISAGE_INCLUDE_DIRS and VISAGE_LIBRARIES variables only in the advanced view + mark_as_advanced(VISAGE_INCLUDE_DIRS VISAGE_LIBRARIES) + +endif (VISAGE_LIBRARIES AND VISAGE_INCLUDE_DIRS) diff --git a/examples/editVoxels.js b/examples/editVoxels.js index c1f0c8dc49..c67ff0dcfa 100644 --- a/examples/editVoxels.js +++ b/examples/editVoxels.js @@ -16,14 +16,7 @@ // Click and drag to create more new voxels in the same direction // -function vLength(v) { - return Math.sqrt(v.x * v.x + v.y * v.y + v.z * v.z); -} - -function vMinus(a, b) { - var rval = { x: a.x - b.x, y: a.y - b.y, z: a.z - b.z }; - return rval; -} +var windowDimensions = Controller.getViewportDimensions(); var NEW_VOXEL_SIZE = 1.0; var NEW_VOXEL_DISTANCE_FROM_CAMERA = 3.0; @@ -76,6 +69,52 @@ var clickSound = new Sound("https://s3-us-west-1.amazonaws.com/highfidelity-publ var audioOptions = new AudioInjectionOptions();
 audioOptions.volume = 0.5; +var editToolsOn = false; // starts out off + + +var voxelPreview = Overlays.addOverlay("cube", { + position: { x: 0, y: 0, z: 0}, + size: 1, + color: { red: 255, green: 0, blue: 0}, + alpha: 1, + solid: false, + visible: false, + lineWidth: 4 + }); + + +// These will be our "overlay IDs" +var swatches = new Array(); +var swatchHeight = 54; +var swatchWidth = 31; +var swatchesWidth = swatchWidth * numColors; +var swatchesX = (windowDimensions.x - swatchesWidth) / 2; +var swatchesY = windowDimensions.y - swatchHeight; + +// create the overlays, position them in a row, set their colors, and for the selected one, use a different source image +// location so that it displays the "selected" marker +for (s = 0; s < numColors; s++) { + var imageFromX = 12 + (s * 27); + var imageFromY = 0; + if (s == whichColor) { + imageFromY = 55; + } + var swatchX = swatchesX + (30 * s); + + swatches[s] = Overlays.addOverlay("image", { + x: swatchX, + y: swatchesY, + width: swatchWidth, + height: swatchHeight, + subImage: { x: imageFromX, y: imageFromY, width: (swatchWidth - 1), height: swatchHeight }, + imageURL: "http://highfidelity-public.s3-us-west-1.amazonaws.com/images/testing-swatches.svg", + color: colors[s], + alpha: 1, + visible: editToolsOn + }); +} + + function setAudioPosition() { var camera = Camera.getPosition(); var forwardVector = Quat.getFront(MyAvatar.orientation); @@ -101,7 +140,141 @@ function fixEulerAngles(eulers) { return rVal; } +var trackLastMouseX = 0; +var trackLastMouseY = 0; +var trackAsDelete = false; +var trackAsRecolor = false; + +function showPreviewVoxel() { + if (editToolsOn) { + var voxelColor; + + var pickRay = Camera.computePickRay(trackLastMouseX, trackLastMouseY); + var intersection = Voxels.findRayIntersection(pickRay); + + if (whichColor == -1) { + // Copy mode - use clicked voxel color + voxelColor = { red: intersection.voxel.red, + green: intersection.voxel.green, + blue: intersection.voxel.blue }; + } else { + voxelColor = { red: colors[whichColor].red, + green: colors[whichColor].green, + blue: colors[whichColor].blue }; + } + + var guidePosition; + + if (trackAsDelete) { + guidePosition = { x: intersection.voxel.x, + y: intersection.voxel.y, + z: intersection.voxel.z }; + Overlays.editOverlay(voxelPreview, { + position: guidePosition, + size: intersection.voxel.s, + visible: true, + color: { red: 255, green: 0, blue: 0 }, + solid: false, + alpha: 1 + }); + } else if (trackAsRecolor) { + guidePosition = { x: intersection.voxel.x - 0.001, + y: intersection.voxel.y - 0.001, + z: intersection.voxel.z - 0.001 }; + + Overlays.editOverlay(voxelPreview, { + position: guidePosition, + size: intersection.voxel.s + 0.002, + visible: true, + color: voxelColor, + solid: true, + alpha: 0.8 + }); + + } else if (!isExtruding) { + guidePosition = { x: intersection.voxel.x, + y: intersection.voxel.y, + z: intersection.voxel.z }; + + if (intersection.face == "MIN_X_FACE") { + guidePosition.x -= intersection.voxel.s; + } else if (intersection.face == "MAX_X_FACE") { + guidePosition.x += intersection.voxel.s; + } else if (intersection.face == "MIN_Y_FACE") { + guidePosition.y -= intersection.voxel.s; + } else if (intersection.face == "MAX_Y_FACE") { + guidePosition.y += intersection.voxel.s; + } else if (intersection.face == "MIN_Z_FACE") { + guidePosition.z -= intersection.voxel.s; + } else if (intersection.face == "MAX_Z_FACE") { + guidePosition.z += intersection.voxel.s; + } + + Overlays.editOverlay(voxelPreview, { + position: guidePosition, + size: intersection.voxel.s, + visible: true, + color: voxelColor, + solid: true, + alpha: 0.7 + }); + } else if (isExtruding) { + Overlays.editOverlay(voxelPreview, { visible: false }); + } + } else { + Overlays.editOverlay(voxelPreview, { visible: false }); + } +} + +function trackMouseEvent(event) { + trackLastMouseX = event.x; + trackLastMouseY = event.y; + trackAsDelete = event.isControl; + trackAsRecolor = event.isShifted; + showPreviewVoxel(); +} + +function trackKeyPressEvent(event) { + if (event.text == "CONTROL") { + trackAsDelete = true; + showPreviewVoxel(); + } + if (event.text == "SHIFT") { + trackAsRecolor = true; + } + showPreviewVoxel(); +} + +function trackKeyReleaseEvent(event) { + if (event.text == "CONTROL") { + trackAsDelete = false; + showPreviewVoxel(); + } + if (event.text == "SHIFT") { + trackAsRecolor = false; + } + + // on TAB release, toggle our tool state + if (event.text == "TAB") { + editToolsOn = !editToolsOn; + moveTools(); + Audio.playSound(clickSound, audioOptions); + } + showPreviewVoxel(); +} + function mousePressEvent(event) { + + // if our tools are off, then don't do anything + if (!editToolsOn) { + return; + } + + if (event.isRightButton) { + // debugging of right button click on mac... + print(">>>> RIGHT BUTTON <<<<<"); + } + trackMouseEvent(event); // used by preview support mouseX = event.x; mouseY = event.y; var pickRay = Camera.computePickRay(event.x, event.y); @@ -118,16 +291,17 @@ function mousePressEvent(event) { // get position for initial azimuth, elevation orbitCenter = intersection.intersection; var orbitVector = Vec3.subtract(cameraPosition, orbitCenter); - orbitRadius = vLength(orbitVector); + orbitRadius = Vec3.length(orbitVector); orbitAzimuth = Math.atan2(orbitVector.z, orbitVector.x); orbitAltitude = Math.asin(orbitVector.y / Vec3.length(orbitVector)); - } else if (event.isRightButton || event.isControl) { + } else if (trackAsDelete || event.isRightButton) { // Delete voxel Voxels.eraseVoxel(intersection.voxel.x, intersection.voxel.y, intersection.voxel.z, intersection.voxel.s); Audio.playSound(deleteSound, audioOptions); + Overlays.editOverlay(voxelPreview, { visible: false }); - } else if (event.isShifted) { + } else if (trackAsRecolor) { // Recolor Voxel Voxels.setVoxel(intersection.voxel.x, intersection.voxel.y, @@ -135,6 +309,7 @@ function mousePressEvent(event) { intersection.voxel.s, colors[whichColor].red, colors[whichColor].green, colors[whichColor].blue); Audio.playSound(changeColorSound, audioOptions); + Overlays.editOverlay(voxelPreview, { visible: false }); } else { // Add voxel on face if (whichColor == -1) { @@ -178,6 +353,7 @@ function mousePressEvent(event) { lastVoxelScale = newVoxel.s; Audio.playSound(addSound, audioOptions); + Overlays.editOverlay(voxelPreview, { visible: false }); dragStart = { x: event.x, y: event.y }; isAdding = true; } @@ -185,42 +361,52 @@ function mousePressEvent(event) { } function keyPressEvent(event) { - key_alt = event.isAlt; - key_shift = event.isShifted; - var nVal = parseInt(event.text); - if (event.text == "0") { - print("Color = Copy"); - whichColor = -1; - Audio.playSound(clickSound, audioOptions); - } else if ((nVal > 0) && (nVal <= numColors)) { - whichColor = nVal - 1; - print("Color = " + (whichColor + 1)); - Audio.playSound(clickSound, audioOptions); - } else if (event.text == "9") { - // Create a brand new 1 meter voxel in front of your avatar - var color = whichColor; - if (color == -1) color = 0; - var newPosition = getNewVoxelPosition(); - var newVoxel = { - x: newPosition.x, - y: newPosition.y , - z: newPosition.z, - s: NEW_VOXEL_SIZE, - red: colors[color].red, - green: colors[color].green, - blue: colors[color].blue }; - Voxels.setVoxel(newVoxel.x, newVoxel.y, newVoxel.z, newVoxel.s, newVoxel.red, newVoxel.green, newVoxel.blue); - setAudioPosition(); - Audio.playSound(addSound, audioOptions); - } else if (event.text == " ") { + // if our tools are off, then don't do anything + if (editToolsOn) { + key_alt = event.isAlt; + key_shift = event.isShifted; + var nVal = parseInt(event.text); + if (event.text == "0") { + print("Color = Copy"); + whichColor = -1; + Audio.playSound(clickSound, audioOptions); + moveTools(); + } else if ((nVal > 0) && (nVal <= numColors)) { + whichColor = nVal - 1; + print("Color = " + (whichColor + 1)); + Audio.playSound(clickSound, audioOptions); + moveTools(); + } else if (event.text == "9") { + // Create a brand new 1 meter voxel in front of your avatar + var color = whichColor; + if (color == -1) color = 0; + var newPosition = getNewVoxelPosition(); + var newVoxel = { + x: newPosition.x, + y: newPosition.y , + z: newPosition.z, + s: NEW_VOXEL_SIZE, + red: colors[color].red, + green: colors[color].green, + blue: colors[color].blue }; + Voxels.setVoxel(newVoxel.x, newVoxel.y, newVoxel.z, newVoxel.s, newVoxel.red, newVoxel.green, newVoxel.blue); + setAudioPosition(); + Audio.playSound(addSound, audioOptions); + } + } + + // do this even if not in edit tools + if (event.text == " ") { // Reset my orientation! var orientation = { x:0, y:0, z:0, w:1 }; Camera.setOrientation(orientation); MyAvatar.orientation = orientation; } + trackKeyPressEvent(event); // used by preview support } function keyReleaseEvent(event) { + trackKeyReleaseEvent(event); // used by preview support key_alt = false; key_shift = false; } @@ -248,7 +434,7 @@ function mouseMoveEvent(event) { var lastVoxelDistance = { x: pickRay.origin.x - lastVoxelPosition.x, y: pickRay.origin.y - lastVoxelPosition.y, z: pickRay.origin.z - lastVoxelPosition.z }; - var distance = vLength(lastVoxelDistance); + var distance = Vec3.length(lastVoxelDistance); var mouseSpot = { x: pickRay.direction.x * distance, y: pickRay.direction.y * distance, z: pickRay.direction.z * distance }; mouseSpot.x += pickRay.origin.x; mouseSpot.y += pickRay.origin.y; @@ -279,9 +465,17 @@ function mouseMoveEvent(event) { } } } + + // update the add voxel/delete voxel overlay preview + trackMouseEvent(event); } function mouseReleaseEvent(event) { + // if our tools are off, then don't do anything + if (!editToolsOn) { + return; + } + if (isOrbiting) { var cameraOrientation = Camera.getOrientation(); var eulers = Quat.safeEulerAngles(cameraOrientation); @@ -296,6 +490,41 @@ function mouseReleaseEvent(event) { isExtruding = false; } +function moveTools() { + swatchesX = (windowDimensions.x - swatchesWidth) / 2; + swatchesY = windowDimensions.y - swatchHeight; + + // create the overlays, position them in a row, set their colors, and for the selected one, use a different source image + // location so that it displays the "selected" marker + for (s = 0; s < numColors; s++) { + var imageFromX = 12 + (s * 27); + var imageFromY = 0; + if (s == whichColor) { + imageFromY = 55; + } + var swatchX = swatchesX + ((swatchWidth - 1) * s); + + Overlays.editOverlay(swatches[s], { + x: swatchX, + y: swatchesY, + subImage: { x: imageFromX, y: imageFromY, width: (swatchWidth - 1), height: swatchHeight }, + color: colors[s], + alpha: 1, + visible: editToolsOn + }); + } +} + + +function update() { + var newWindowDimensions = Controller.getViewportDimensions(); + if (newWindowDimensions.x != windowDimensions.x || newWindowDimensions.y != windowDimensions.y) { + windowDimensions = newWindowDimensions; + print("window resized..."); + moveTools(); + } +} + Controller.mousePressEvent.connect(mousePressEvent); Controller.mouseReleaseEvent.connect(mouseReleaseEvent); Controller.mouseMoveEvent.connect(mouseMoveEvent); @@ -303,5 +532,15 @@ Controller.keyPressEvent.connect(keyPressEvent); Controller.keyReleaseEvent.connect(keyReleaseEvent); function scriptEnding() { + Overlays.deleteOverlay(voxelPreview); + for (s = 0; s < numColors; s++) { + Overlays.deleteOverlay(swatches[s]); + } } Script.scriptEnding.connect(scriptEnding); + + +Script.willSendVisualDataCallback.connect(update); + + + diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 8e96006828..32e75c9f34 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -11,6 +11,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../cmake set(FACESHIFT_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/faceshift) set(LIBOVR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/LibOVR) set(SIXENSE_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/Sixense) +set(VISAGE_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/visage) if (DEFINED ENV{JOB_ID}) set(BUILD_SEQ $ENV{JOB_ID}) @@ -138,9 +139,10 @@ find_package(Faceshift) find_package(GLM REQUIRED) find_package(LibOVR) find_package(Sixense) +find_package(Visage) find_package(ZLIB) -# likewise with Sixense library for Razer Hydra +# include the Sixense library for Razer Hydra if available if (SIXENSE_FOUND AND NOT DISABLE_SIXENSE) add_definitions(-DHAVE_SIXENSE) include_directories(SYSTEM ${SIXENSE_INCLUDE_DIRS}) @@ -150,6 +152,21 @@ if (SIXENSE_FOUND AND NOT DISABLE_SIXENSE) target_link_libraries(${TARGET_NAME} ${SIXENSE_LIBRARIES}) endif (SIXENSE_FOUND AND NOT DISABLE_SIXENSE) +# likewise with Visage library for webcam feature tracking +if (VISAGE_FOUND AND NOT DISABLE_VISAGE) + add_definitions(-DHAVE_VISAGE -DVISAGE_STATIC) + include_directories(SYSTEM ${VISAGE_INCLUDE_DIRS}) + if (APPLE) + add_definitions(-DMAC_OS_X) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-comment -isystem ${VISAGE_INCLUDE_DIRS}") + find_library(AVFoundation AVFoundation) + find_library(CoreMedia CoreMedia) + find_library(NEW_STD_LIBRARY libc++abi.dylib /usr/lib/) + target_link_libraries(${TARGET_NAME} ${AVFoundation} ${CoreMedia} ${NEW_STD_LIBRARY}) + endif (APPLE) + target_link_libraries(${TARGET_NAME} ${VISAGE_LIBRARIES}) +endif (VISAGE_FOUND AND NOT DISABLE_VISAGE) + # and with LibOVR for Oculus Rift if (LIBOVR_FOUND AND NOT DISABLE_LIBOVR) add_definitions(-DHAVE_LIBOVR) diff --git a/interface/external/visage/readme.txt b/interface/external/visage/readme.txt new file mode 100644 index 0000000000..8aa28f81c4 --- /dev/null +++ b/interface/external/visage/readme.txt @@ -0,0 +1,14 @@ + +Instructions for adding the Visage driver to Interface +Andrzej Kapolka, February 11, 2014 + +1. Copy the Visage sdk folders (lib, include, dependencies) into the interface/external/visage folder. + This readme.txt should be there as well. + +2. Copy the Visage configuration data folder (visageSDK-MacOS/Samples/MacOSX/data/) to interface/resources/visage + (i.e., so that interface/resources/visage/candide3.wfm is accessible) + +3. Copy the Visage license file to interface/resources/visage/license.vlc. + +4. Delete your build directory, run cmake and build, and you should be all set. + diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7c8d6e3a34..766f21626b 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2041,6 +2041,15 @@ void Application::updateFaceshift() { } } +void Application::updateVisage() { + + bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); + PerformanceWarning warn(showWarnings, "Application::updateVisage()"); + + // Update Visage + _visage.update(); +} + void Application::updateMyAvatarLookAtPosition(glm::vec3& lookAtSpot) { bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); @@ -2061,13 +2070,25 @@ void Application::updateMyAvatarLookAtPosition(glm::vec3& lookAtSpot) { } lookAtSpot = _mouseRayOrigin + _mouseRayDirection * distance; } + bool trackerActive = false; + float eyePitch, eyeYaw; if (_faceshift.isActive()) { + eyePitch = _faceshift.getEstimatedEyePitch(); + eyeYaw = _faceshift.getEstimatedEyeYaw(); + trackerActive = true; + + } else if (_visage.isActive()) { + eyePitch = _visage.getEstimatedEyePitch(); + eyeYaw = _visage.getEstimatedEyeYaw(); + trackerActive = true; + } + if (trackerActive) { // deflect using Faceshift gaze data glm::vec3 origin = _myAvatar->getHead()->calculateAverageEyePosition(); float pitchSign = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? -1.0f : 1.0f; float deflection = Menu::getInstance()->getFaceshiftEyeDeflection(); lookAtSpot = origin + _myCamera.getRotation() * glm::quat(glm::radians(glm::vec3( - _faceshift.getEstimatedEyePitch() * pitchSign * deflection, _faceshift.getEstimatedEyeYaw() * deflection, 0.0f))) * + eyePitch * pitchSign * deflection, eyeYaw * deflection, 0.0f))) * glm::inverse(_myCamera.getRotation()) * (lookAtSpot - origin); } _myAvatar->getHead()->setLookAtPosition(lookAtSpot); @@ -2318,6 +2339,7 @@ void Application::update(float deltaTime) { glm::vec3 lookAtSpot; updateFaceshift(); + updateVisage(); _myAvatar->updateLookAtTargetAvatar(lookAtSpot); updateMyAvatarLookAtPosition(lookAtSpot); @@ -3821,6 +3843,7 @@ void Application::resetSensors() { _mouseY = _glWidget->height() / 2; _faceshift.reset(); + _visage.reset(); if (OculusManager::isConnected()) { OculusManager::reset(); diff --git a/interface/src/Application.h b/interface/src/Application.h index 0b6907e0f5..06a19f13b1 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -54,6 +54,7 @@ #include "avatar/Profile.h" #include "devices/Faceshift.h" #include "devices/SixenseManager.h" +#include "devices/Visage.h" #include "renderer/AmbientOcclusionEffect.h" #include "renderer/GeometryCache.h" #include "renderer/GlowEffect.h" @@ -160,6 +161,7 @@ public: const glm::vec3& getMouseRayOrigin() const { return _mouseRayOrigin; } const glm::vec3& getMouseRayDirection() const { return _mouseRayDirection; } Faceshift* getFaceshift() { return &_faceshift; } + Visage* getVisage() { return &_visage; } SixenseManager* getSixenseManager() { return &_sixenseManager; } BandwidthMeter* getBandwidthMeter() { return &_bandwidthMeter; } QSettings* getSettings() { return _settings; } @@ -283,6 +285,7 @@ private: // Various helper functions called during update() void updateMouseRay(); void updateFaceshift(); + void updateVisage(); void updateMyAvatarLookAtPosition(glm::vec3& lookAtSpot); void updateHoverVoxels(float deltaTime, float& distance, BoxFace& face); void updateMouseVoxels(float deltaTime, float& distance, BoxFace& face); @@ -382,6 +385,7 @@ private: Profile _profile; // The data-server linked profile for this user Faceshift _faceshift; + Visage _visage; SixenseManager _sixenseManager; QStringList _activeScripts; diff --git a/interface/src/ControllerScriptingInterface.cpp b/interface/src/ControllerScriptingInterface.cpp index b3d6170bff..b60615f124 100644 --- a/interface/src/ControllerScriptingInterface.cpp +++ b/interface/src/ControllerScriptingInterface.cpp @@ -250,3 +250,7 @@ void ControllerScriptingInterface::releaseJoystick(int joystickIndex) { } } +glm::vec2 ControllerScriptingInterface::getViewportDimensions() const { + QGLWidget* widget = Application::getInstance()->getGLWidget(); + return glm::vec2(widget->width(), widget->height()); +} diff --git a/interface/src/ControllerScriptingInterface.h b/interface/src/ControllerScriptingInterface.h index f0a50559f9..6fe5a60fa4 100644 --- a/interface/src/ControllerScriptingInterface.h +++ b/interface/src/ControllerScriptingInterface.h @@ -74,6 +74,8 @@ public slots: virtual void captureJoystick(int joystickIndex); virtual void releaseJoystick(int joystickIndex); + virtual glm::vec2 getViewportDimensions() const; + private: const PalmData* getPrimaryPalm() const; const PalmData* getPalm(int palmIndex) const; diff --git a/interface/src/VoxelSystem.cpp b/interface/src/VoxelSystem.cpp index a3352f36e7..693a010182 100644 --- a/interface/src/VoxelSystem.cpp +++ b/interface/src/VoxelSystem.cpp @@ -2105,7 +2105,7 @@ void VoxelSystem::hideOutOfView(bool forceFullFrustum) { bool VoxelSystem::hideAllSubTreeOperation(OctreeElement* element, void* extraData) { VoxelTreeElement* voxel = (VoxelTreeElement*)element; hideOutOfViewArgs* args = (hideOutOfViewArgs*)extraData; - + // If we've culled at least once, then we will use the status of this voxel in the last culled frustum to determine // how to proceed. If we've never culled, then we just consider all these voxels to be UNKNOWN so that we will not // consider that case. @@ -2141,7 +2141,7 @@ bool VoxelSystem::hideAllSubTreeOperation(OctreeElement* element, void* extraDat bool VoxelSystem::showAllSubTreeOperation(OctreeElement* element, void* extraData) { VoxelTreeElement* voxel = (VoxelTreeElement*)element; hideOutOfViewArgs* args = (hideOutOfViewArgs*)extraData; - + // If we've culled at least once, then we will use the status of this voxel in the last culled frustum to determine // how to proceed. If we've never culled, then we just consider all these voxels to be UNKNOWN so that we will not // consider that case. @@ -2184,7 +2184,7 @@ bool VoxelSystem::showAllSubTreeOperation(OctreeElement* element, void* extraDat bool VoxelSystem::hideOutOfViewOperation(OctreeElement* element, void* extraData) { VoxelTreeElement* voxel = (VoxelTreeElement*)element; hideOutOfViewArgs* args = (hideOutOfViewArgs*)extraData; - + // If we're still recursing the tree using this operator, then we don't know if we're inside or outside... // so before we move forward we need to determine our frustum location ViewFrustum::location inFrustum = voxel->inFrustum(args->thisViewFrustum); @@ -2201,7 +2201,6 @@ bool VoxelSystem::hideOutOfViewOperation(OctreeElement* element, void* extraData // ok, now do some processing for this node... switch (inFrustum) { case ViewFrustum::OUTSIDE: { - // If this node is outside the current view, then we might want to hide it... unless it was previously OUTSIDE, // if it was previously outside, then we can safely assume it's already hidden, and we can also safely assume // that all of it's children are outside both of our views, in which case we can just stop recursing... @@ -2215,12 +2214,10 @@ bool VoxelSystem::hideOutOfViewOperation(OctreeElement* element, void* extraData // we need to hide it. Additionally we know that ALL of it's children are also fully OUTSIDE so we can recurse // the children and simply mark them as hidden args->tree->recurseNodeWithOperation(voxel, hideAllSubTreeOperation, args ); - return false; } break; case ViewFrustum::INSIDE: { - // If this node is INSIDE the current view, then we might want to show it... unless it was previously INSIDE, // if it was previously INSIDE, then we can safely assume it's already shown, and we can also safely assume // that all of it's children are INSIDE both of our views, in which case we can just stop recursing... @@ -2234,12 +2231,10 @@ bool VoxelSystem::hideOutOfViewOperation(OctreeElement* element, void* extraData // we need to show it. Additionally we know that ALL of it's children are also fully INSIDE so we can recurse // the children and simply mark them as visible (as appropriate based on LOD) args->tree->recurseNodeWithOperation(voxel, showAllSubTreeOperation, args); - return false; } break; case ViewFrustum::INTERSECT: { args->nodesScanned++; - // If this node INTERSECTS the current view, then we might want to show it... unless it was previously INSIDE // the last known view, in which case it will already be visible, and we know that all it's children are also // previously INSIDE and visible. So in this case stop recursing @@ -2253,8 +2248,15 @@ bool VoxelSystem::hideOutOfViewOperation(OctreeElement* element, void* extraData // if the child node INTERSECTs the view, then we want to check to see if it thinks it should render // if it should render but is missing it's VBO index, then we want to flip it on, and we can stop recursing from // here because we know will block any children anyway + + float voxelSizeScale = Menu::getInstance()->getVoxelSizeScale(); + int boundaryLevelAdjust = Menu::getInstance()->getBoundaryLevelAdjust(); + bool shouldRender = voxel->calculateShouldRender(&args->thisViewFrustum, voxelSizeScale, boundaryLevelAdjust); + voxel->setShouldRender(shouldRender); + if (voxel->getShouldRender() && !voxel->isKnownBufferIndex()) { voxel->setDirtyBit(); // will this make it draw? + voxel->markWithChangedTime(); // both are needed to force redraw args->nodesShown++; return false; } @@ -2267,7 +2269,6 @@ bool VoxelSystem::hideOutOfViewOperation(OctreeElement* element, void* extraData } break; } // switch - return true; // keep going! } diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index ddb0660364..5c6100764a 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -62,24 +62,20 @@ void Head::simulate(float deltaTime, bool isMine) { // Update audio trailing average for rendering facial animations Faceshift* faceshift = Application::getInstance()->getFaceshift(); + Visage* visage = Application::getInstance()->getVisage(); if (isMine) { - _isFaceshiftConnected = faceshift->isActive(); + _isFaceshiftConnected = false; + if (faceshift->isActive()) { + _blendshapeCoefficients = faceshift->getBlendshapeCoefficients(); + _isFaceshiftConnected = true; + + } else if (visage->isActive()) { + _blendshapeCoefficients = visage->getBlendshapeCoefficients(); + _isFaceshiftConnected = true; + } } - if (isMine && faceshift->isActive()) { - const float EYE_OPEN_SCALE = 0.5f; - _leftEyeBlink = faceshift->getLeftBlink() - EYE_OPEN_SCALE * faceshift->getLeftEyeOpen(); - _rightEyeBlink = faceshift->getRightBlink() - EYE_OPEN_SCALE * faceshift->getRightEyeOpen(); - - // set these values based on how they'll be used. if we use faceshift in the long term, we'll want a complete - // mapping between their blendshape coefficients and our avatar features - const float MOUTH_SIZE_SCALE = 2500.0f; - _averageLoudness = faceshift->getMouthSize() * faceshift->getMouthSize() * MOUTH_SIZE_SCALE; - const float BROW_HEIGHT_SCALE = 0.005f; - _browAudioLift = faceshift->getBrowUpCenter() * BROW_HEIGHT_SCALE; - _blendshapeCoefficients = faceshift->getBlendshapeCoefficients(); - - } else if (!_isFaceshiftConnected) { + if (!_isFaceshiftConnected) { // Update eye saccades const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; const float AVERAGE_SACCADE_INTERVAL = 4.0f; diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index e7f7dd236f..4e8aab600c 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -340,24 +340,34 @@ const float MAX_PITCH = 90.0f; // Update avatar head rotation with sensor data void MyAvatar::updateFromGyros(float deltaTime) { Faceshift* faceshift = Application::getInstance()->getFaceshift(); + Visage* visage = Application::getInstance()->getVisage(); glm::vec3 estimatedPosition, estimatedRotation; - Head* head = getHead(); - + bool trackerActive = false; if (faceshift->isActive()) { estimatedPosition = faceshift->getHeadTranslation(); estimatedRotation = safeEulerAngles(faceshift->getHeadRotation()); + trackerActive = true; + + } else if (visage->isActive()) { + estimatedPosition = visage->getHeadTranslation(); + estimatedRotation = safeEulerAngles(visage->getHeadRotation()); + trackerActive = true; + } + + Head* head = getHead(); + if (trackerActive) { // Rotate the body if the head is turned beyond the screen if (Menu::getInstance()->isOptionChecked(MenuOption::TurnWithHead)) { - const float FACESHIFT_YAW_TURN_SENSITIVITY = 0.5f; - const float FACESHIFT_MIN_YAW_TURN = 15.f; - const float FACESHIFT_MAX_YAW_TURN = 50.f; - if ( (fabs(estimatedRotation.y) > FACESHIFT_MIN_YAW_TURN) && - (fabs(estimatedRotation.y) < FACESHIFT_MAX_YAW_TURN) ) { + const float TRACKER_YAW_TURN_SENSITIVITY = 0.5f; + const float TRACKER_MIN_YAW_TURN = 15.f; + const float TRACKER_MAX_YAW_TURN = 50.f; + if ( (fabs(estimatedRotation.y) > TRACKER_MIN_YAW_TURN) && + (fabs(estimatedRotation.y) < TRACKER_MAX_YAW_TURN) ) { if (estimatedRotation.y > 0.f) { - _bodyYawDelta += (estimatedRotation.y - FACESHIFT_MIN_YAW_TURN) * FACESHIFT_YAW_TURN_SENSITIVITY; + _bodyYawDelta += (estimatedRotation.y - TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; } else { - _bodyYawDelta += (estimatedRotation.y + FACESHIFT_MIN_YAW_TURN) * FACESHIFT_YAW_TURN_SENSITIVITY; + _bodyYawDelta += (estimatedRotation.y + TRACKER_MIN_YAW_TURN) * TRACKER_YAW_TURN_SENSITIVITY; } } } diff --git a/interface/src/devices/Visage.cpp b/interface/src/devices/Visage.cpp new file mode 100644 index 0000000000..26e7993ecb --- /dev/null +++ b/interface/src/devices/Visage.cpp @@ -0,0 +1,164 @@ +// +// Visage.cpp +// interface +// +// Created by Andrzej Kapolka on 2/11/14. +// Copyright (c) 2014 High Fidelity, Inc. All rights reserved. +// + +#include + +#include + +#ifdef HAVE_VISAGE +#include +#endif + +#include "Visage.h" +#include "renderer/FBXReader.h" + +namespace VisageSDK { +#ifdef WIN32 + void __declspec(dllimport) initializeLicenseManager(char* licenseKeyFileName); +#else + void initializeLicenseManager(char* licenseKeyFileName); +#endif +} + +using namespace VisageSDK; + +const glm::vec3 DEFAULT_HEAD_ORIGIN(0.0f, 0.0f, 0.7f); + +Visage::Visage() : + _active(false), + _headOrigin(DEFAULT_HEAD_ORIGIN), + _estimatedEyePitch(0.0f), + _estimatedEyeYaw(0.0f), + _leftInnerBrowIndex(0), + _rightInnerBrowIndex(0) { + +#ifdef HAVE_VISAGE + switchToResourcesParentIfRequired(); + QByteArray licensePath = "resources/visage/license.vlc"; + initializeLicenseManager(licensePath.data()); + _tracker = new VisageTracker2("resources/visage/Facial Features Tracker - Asymmetric.cfg"); + if (_tracker->trackFromCam()) { + _data = new FaceData(); + + } else { + delete _tracker; + _tracker = NULL; + } +#endif +} + +Visage::~Visage() { +#ifdef HAVE_VISAGE + if (_tracker) { + _tracker->stop(); + delete _tracker; + delete _data; + } +#endif +} + +static int leftEyeBlinkIndex = 0; +static int rightEyeBlinkIndex = 1; +static int centerBrowIndex = 16; + +static QHash createBlendshapeIndices() { + QHash blendshapeMap; + blendshapeMap.insert("Sneer", "au_nose_wrinkler"); + blendshapeMap.insert("JawFwd", "au_jaw_z_push"); + blendshapeMap.insert("JawLeft", "au_jaw_x_push"); + blendshapeMap.insert("JawOpen", "au_jaw_drop"); + blendshapeMap.insert("LipsLowerDown", "au_lower_lip_drop"); + blendshapeMap.insert("LipsUpperUp", "au_upper_lip_raiser"); + blendshapeMap.insert("LipsStretch_L", "au_lip_stretcher_left"); + blendshapeMap.insert("BrowsU_L", "au_left_outer_brow_raiser"); + blendshapeMap.insert("BrowsU_C", "au_left_inner_brow_raiser"); + blendshapeMap.insert("BrowsD_L", "au_left_brow_lowerer"); + blendshapeMap.insert("LipsStretch_R", "au_lip_stretcher_right"); + blendshapeMap.insert("BrowsU_R", "au_right_outer_brow_raiser"); + blendshapeMap.insert("BrowsU_C", "au_right_inner_brow_raiser"); + blendshapeMap.insert("BrowsD_R", "au_right_brow_lowerer"); + + QHash blendshapeIndices; + for (int i = 0;; i++) { + QByteArray blendshape = FACESHIFT_BLENDSHAPES[i]; + if (blendshape.isEmpty()) { + break; + } + if (blendshape == "EyeBlink_L") { + leftEyeBlinkIndex = i; + + } else if (blendshape == "EyeBlink_R") { + rightEyeBlinkIndex = i; + + } else if (blendshape == "BrowsU_C") { + centerBrowIndex = i; + } + QByteArray mapping = blendshapeMap.value(blendshape); + if (!mapping.isEmpty()) { + blendshapeIndices.insert(mapping, i + 1); + } + } + + return blendshapeIndices; +} + +static const QHash& getBlendshapeIndices() { + static QHash blendshapeIndices = createBlendshapeIndices(); + return blendshapeIndices; +} + +const float TRANSLATION_SCALE = 20.0f; + +void Visage::update() { +#ifdef HAVE_VISAGE + _active = (_tracker && _tracker->getTrackingData(_data) == TRACK_STAT_OK); + if (!_active) { + return; + } + _headRotation = glm::quat(glm::vec3(-_data->faceRotation[0], -_data->faceRotation[1], _data->faceRotation[2])); + _headTranslation = (glm::vec3(_data->faceTranslation[0], _data->faceTranslation[1], _data->faceTranslation[2]) - + _headOrigin) * TRANSLATION_SCALE; + _estimatedEyePitch = glm::degrees(-_data->gazeDirection[1]); + _estimatedEyeYaw = glm::degrees(-_data->gazeDirection[0]); + + if (_blendshapeIndices.isEmpty()) { + _blendshapeIndices.resize(_data->actionUnitCount); + int maxIndex = -1; + for (int i = 0; i < _data->actionUnitCount; i++) { + QByteArray name = _data->actionUnitsNames[i]; + if (name == "au_left_inner_brow_raiser") { + _leftInnerBrowIndex = i; + } else if (name == "au_right_inner_brow_raiser") { + _rightInnerBrowIndex = i; + } + int index = getBlendshapeIndices().value(name) - 1; + maxIndex = qMax(maxIndex, _blendshapeIndices[i] = index); + } + _blendshapeCoefficients.resize(maxIndex + 1); + } + + qFill(_blendshapeCoefficients.begin(), _blendshapeCoefficients.end(), 0.0f); + for (int i = 0; i < _data->actionUnitCount; i++) { + if (!_data->actionUnitsUsed[i]) { + continue; + } + int index = _blendshapeIndices.at(i); + if (index != -1) { + _blendshapeCoefficients[index] = _data->actionUnits[i]; + } + } + _blendshapeCoefficients[leftEyeBlinkIndex] = 1.0f - _data->eyeClosure[1]; + _blendshapeCoefficients[rightEyeBlinkIndex] = 1.0f - _data->eyeClosure[0]; + _blendshapeCoefficients[centerBrowIndex] = (_data->actionUnits[_leftInnerBrowIndex] + + _data->actionUnits[_rightInnerBrowIndex]) * 0.5f; +#endif +} + +void Visage::reset() { + _headOrigin += _headTranslation / TRANSLATION_SCALE; +} diff --git a/interface/src/devices/Visage.h b/interface/src/devices/Visage.h new file mode 100644 index 0000000000..124ef29e3b --- /dev/null +++ b/interface/src/devices/Visage.h @@ -0,0 +1,65 @@ +// +// Visage.h +// interface +// +// Created by Andrzej Kapolka on 2/11/14. +// Copyright (c) 2014 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__Visage__ +#define __interface__Visage__ + +#include + +#include + +#include +#include + +namespace VisageSDK { + class VisageTracker2; + struct FaceData; +} + +/// Handles input from the Visage webcam feature tracking software. +class Visage { +public: + + Visage(); + ~Visage(); + + bool isActive() const { return _active; } + + const glm::quat& getHeadRotation() const { return _headRotation; } + const glm::vec3& getHeadTranslation() const { return _headTranslation; } + + float getEstimatedEyePitch() const { return _estimatedEyePitch; } + float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } + + const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + + void update(); + void reset(); + +private: + + VisageSDK::VisageTracker2* _tracker; + VisageSDK::FaceData* _data; + + bool _active; + glm::quat _headRotation; + glm::vec3 _headTranslation; + + glm::vec3 _headOrigin; + + float _estimatedEyePitch; + float _estimatedEyeYaw; + + std::vector _blendshapeCoefficients; + + QVector _blendshapeIndices; + int _leftInnerBrowIndex; + int _rightInnerBrowIndex; +}; + +#endif /* defined(__interface__Visage__) */ diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 8efb23c98c..b89d0954b4 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -21,6 +21,9 @@ class FBXNode; typedef QList FBXNodeList; +/// The names of the blendshapes expected by Faceshift, terminated with an empty string. +extern const char* FACESHIFT_BLENDSHAPES[]; + /// A node within an FBX document. class FBXNode { public: diff --git a/libraries/script-engine/src/AbstractControllerScriptingInterface.h b/libraries/script-engine/src/AbstractControllerScriptingInterface.h index d9878d0b71..1878edd4d6 100644 --- a/libraries/script-engine/src/AbstractControllerScriptingInterface.h +++ b/libraries/script-engine/src/AbstractControllerScriptingInterface.h @@ -52,6 +52,10 @@ public slots: virtual void captureWheelEvents() = 0; virtual void releaseWheelEvents() = 0; + virtual void captureJoystick(int joystickIndex) = 0; + virtual void releaseJoystick(int joystickIndex) = 0; + + virtual glm::vec2 getViewportDimensions() const = 0; signals: void keyPressEvent(const KeyEvent& event); diff --git a/libraries/script-engine/src/EventTypes.cpp b/libraries/script-engine/src/EventTypes.cpp index 963912fd34..8fac2bcd2a 100644 --- a/libraries/script-engine/src/EventTypes.cpp +++ b/libraries/script-engine/src/EventTypes.cpp @@ -102,6 +102,8 @@ KeyEvent::KeyEvent(const QKeyEvent& event) { text = "END"; } else if (key == Qt::Key_Help) { text = "HELP"; + } else if (key == Qt::Key_CapsLock) { + text = "CAPS LOCK"; } } @@ -208,6 +210,8 @@ void keyEventFromScriptValue(const QScriptValue& object, KeyEvent& event) { event.key = Qt::Key_End; } else if (event.text.toUpper() == "HELP") { event.key = Qt::Key_Help; + } else if (event.text.toUpper() == "CAPS LOCK") { + event.key = Qt::Key_CapsLock; } else { event.key = event.text.at(0).unicode(); }