mirror of
https://github.com/Armored-Dragon/overte.git
synced 2025-03-11 16:13:16 +01:00
Merge pull request #16517 from hyperlogic/feature/facecap-osc-plugin
Input plugin for streaming blendshapes from an iPhone
This commit is contained in:
Anthony Thibault
GitHub
commit
edfad97b64
17 changed files with 950 additions and 24 deletions
4
cmake/macros/TargetLiblo.cmake
Normal file
4
cmake/macros/TargetLiblo.cmake
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
macro(target_liblo)
|
||||
find_library(LIBLO LIBLO)
|
||||
target_link_libraries(${TARGET_NAME} ${LIBLO})
|
||||
endmacro()
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
Source: hifi-client-deps
|
||||
Version: 0
|
||||
Version: 0.1
|
||||
Description: Collected dependencies for High Fidelity applications
|
||||
Build-Depends: hifi-deps, glslang, nlohmann-json, openvr (windows), quazip (!android), sdl2 (!android), spirv-cross (!android), spirv-tools (!android), vulkanmemoryallocator
|
||||
Build-Depends: hifi-deps, glslang, nlohmann-json, openvr (windows), quazip (!android), sdl2 (!android), spirv-cross (!android), spirv-tools (!android), vulkanmemoryallocator, liblo (windows)
|
||||
|
|
|
|||
3
cmake/ports/liblo/CONTROL
Normal file
3
cmake/ports/liblo/CONTROL
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
Source: liblo
|
||||
Version: 0.30
|
||||
Description: liblo is an implementation of the Open Sound Control protocol for POSIX systems
|
||||
36
cmake/ports/liblo/portfile.cmake
Normal file
36
cmake/ports/liblo/portfile.cmake
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
include(vcpkg_common_functions)
|
||||
|
||||
vcpkg_from_github(
|
||||
OUT_SOURCE_PATH SOURCE_PATH
|
||||
REPO radarsat1/liblo
|
||||
REF 0.30
|
||||
SHA512 d36c141c513f869e6d1963bd0d584030038019b8be0b27bb9a684722b6e7a38e942ad2ee7c2e67ac13b965560937aad97259435ed86034aa2dc8cb92d23845d8
|
||||
HEAD_REF master
|
||||
)
|
||||
|
||||
vcpkg_configure_cmake(
|
||||
SOURCE_PATH ${SOURCE_PATH}/cmake
|
||||
PREFER_NINJA # Disable this option if project cannot be built with Ninja
|
||||
OPTIONS -DTHREADING=1
|
||||
)
|
||||
|
||||
vcpkg_install_cmake()
|
||||
|
||||
# Install needed files into package directory
|
||||
vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/liblo)
|
||||
|
||||
file(INSTALL ${CURRENT_PACKAGES_DIR}/bin/oscsend.exe DESTINATION ${CURRENT_PACKAGES_DIR}/tools/liblo)
|
||||
file(INSTALL ${CURRENT_PACKAGES_DIR}/bin/oscdump.exe DESTINATION ${CURRENT_PACKAGES_DIR}/tools/liblo)
|
||||
vcpkg_copy_tool_dependencies(${CURRENT_PACKAGES_DIR}/tools/liblo)
|
||||
|
||||
# Remove unnecessary files
|
||||
file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
|
||||
file(REMOVE ${CURRENT_PACKAGES_DIR}/bin/oscsend.exe ${CURRENT_PACKAGES_DIR}/bin/oscdump.exe)
|
||||
file(REMOVE ${CURRENT_PACKAGES_DIR}/debug/bin/oscsend.exe ${CURRENT_PACKAGES_DIR}/debug/bin/oscdump.exe)
|
||||
|
||||
if(VCPKG_LIBRARY_LINKAGE STREQUAL static)
|
||||
file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/bin ${CURRENT_PACKAGES_DIR}/debug/bin)
|
||||
endif()
|
||||
|
||||
# Handle copyright
|
||||
file(INSTALL ${SOURCE_PATH}/COPYING DESTINATION ${CURRENT_PACKAGES_DIR}/share/liblo RENAME copyright)
|
||||
|
|
@ -82,8 +82,9 @@ endif()
|
|||
self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/builds/vcpkg-win32-client.zip?versionId=tSFzbw01VkkVFeRQ6YuAY4dro2HxJR9U'
|
||||
self.vcpkgHash = 'a650db47a63ccdc9904b68ddd16af74772e7e78170b513ea8de5a3b47d032751a3b73dcc7526d88bcb500753ea3dd9880639ca842bb176e2bddb1710f9a58cd3'
|
||||
self.hostTriplet = 'x64-windows'
|
||||
if ('CI_BUILD' in os.environ) and os.environ["CI_BUILD"] == "Github" and (not self.noClean):
|
||||
self.prebuiltArchive = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/builds/vcpkg-win32.zip?versionId=LtGKnBydCxteY3Ub1W_UNBN5sH.Ccp5g"
|
||||
# Don't use prebuilt on windows, because it is out of date with the inclusion of liblo.
|
||||
# if ('CI_BUILD' in os.environ) and os.environ["CI_BUILD"] == "Github" and (not self.noClean):
|
||||
# self.prebuiltArchive = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/builds/vcpkg-win32.zip?versionId=LtGKnBydCxteY3Ub1W_UNBN5sH.Ccp5g"
|
||||
elif 'Darwin' == system:
|
||||
self.exe = os.path.join(self.path, 'vcpkg')
|
||||
self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '--allowAppleClang' ]
|
||||
|
|
|
|||
71
interface/resources/controllers/osc.json
Normal file
71
interface/resources/controllers/osc.json
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
{
|
||||
"name": "OSC to Standard",
|
||||
"channels": [
|
||||
{ "from": "OSC.Head", "to" : "Standard.Head" },
|
||||
{ "from": "OSC.LeftEye", "to" : "Standard.LeftEye" },
|
||||
{ "from": "OSC.RightEye", "to" : "Standard.RightEye" },
|
||||
|
||||
{ "from": "OSC.EyeBlink_L", "to": "Standard.EyeBlink_L" },
|
||||
{ "from": "OSC.EyeBlink_R", "to": "Standard.EyeBlink_R" },
|
||||
{ "from": "OSC.EyeSquint_L", "to": "Standard.EyeSquint_L" },
|
||||
{ "from": "OSC.EyeSquint_R", "to": "Standard.EyeSquint_R" },
|
||||
{ "from": "OSC.EyeDown_L", "to": "Standard.EyeDown_L" },
|
||||
{ "from": "OSC.EyeDown_R", "to": "Standard.EyeDown_R" },
|
||||
{ "from": "OSC.EyeIn_L", "to": "Standard.EyeIn_L" },
|
||||
{ "from": "OSC.EyeIn_R", "to": "Standard.EyeIn_R" },
|
||||
{ "from": "OSC.EyeOpen_L", "to": "Standard.EyeOpen_L" },
|
||||
{ "from": "OSC.EyeOpen_R", "to": "Standard.EyeOpen_R" },
|
||||
{ "from": "OSC.EyeOut_L", "to": "Standard.EyeOut_L" },
|
||||
{ "from": "OSC.EyeOut_R", "to": "Standard.EyeOut_R" },
|
||||
{ "from": "OSC.EyeUp_L", "to": "Standard.EyeUp_L" },
|
||||
{ "from": "OSC.EyeUp_R", "to": "Standard.EyeUp_R" },
|
||||
{ "from": "OSC.BrowsD_L", "to": "Standard.BrowsD_L" },
|
||||
{ "from": "OSC.BrowsD_R", "to": "Standard.BrowsD_R" },
|
||||
{ "from": "OSC.BrowsU_C", "to": "Standard.BrowsU_C" },
|
||||
{ "from": "OSC.BrowsU_L", "to": "Standard.BrowsU_L" },
|
||||
{ "from": "OSC.BrowsU_R", "to": "Standard.BrowsU_R" },
|
||||
{ "from": "OSC.JawFwd", "to": "Standard.JawFwd" },
|
||||
{ "from": "OSC.JawLeft", "to": "Standard.JawLeft" },
|
||||
{ "from": "OSC.JawOpen", "to": "Standard.JawOpen" },
|
||||
{ "from": "OSC.JawRight", "to": "Standard.JawRight" },
|
||||
{ "from": "OSC.MouthLeft", "to": "Standard.MouthLeft" },
|
||||
{ "from": "OSC.MouthRight", "to": "Standard.MouthRight" },
|
||||
{ "from": "OSC.MouthFrown_L", "to": "Standard.MouthFrown_L" },
|
||||
{ "from": "OSC.MouthFrown_R", "to": "Standard.MouthFrown_R" },
|
||||
{ "from": "OSC.MouthSmile_L", "to": "Standard.MouthSmile_L" },
|
||||
{ "from": "OSC.MouthSmile_R", "to": "Standard.MouthSmile_R" },
|
||||
{ "from": "OSC.MouthDimple_L", "to": "Standard.MouthDimple_L" },
|
||||
{ "from": "OSC.MouthDimple_R", "to": "Standard.MouthDimple_R" },
|
||||
{ "from": "OSC.LipsStretch_L", "to": "Standard.LipsStretch_L" },
|
||||
{ "from": "OSC.LipsStretch_R", "to": "Standard.LipsStretch_R" },
|
||||
{ "from": "OSC.LipsUpperClose", "to": "Standard.LipsUpperClose" },
|
||||
{ "from": "OSC.LipsLowerClose", "to": "Standard.LipsLowerClose" },
|
||||
{ "from": "OSC.LipsFunnel", "to": "Standard.LipsFunnel" },
|
||||
{ "from": "OSC.LipsPucker", "to": "Standard.LipsPucker" },
|
||||
{ "from": "OSC.Puff", "to": "Standard.Puff" },
|
||||
{ "from": "OSC.CheekSquint_L", "to": "Standard.CheekSquint_L" },
|
||||
{ "from": "OSC.CheekSquint_R", "to": "Standard.CheekSquint_R" },
|
||||
{ "from": "OSC.MouthClose", "to": "Standard.MouthClose" },
|
||||
{ "from": "OSC.MouthUpperUp_L", "to": "Standard.MouthUpperUp_L" },
|
||||
{ "from": "OSC.MouthUpperUp_R", "to": "Standard.MouthUpperUp_R" },
|
||||
{ "from": "OSC.MouthLowerDown_L", "to": "Standard.MouthLowerDown_L" },
|
||||
{ "from": "OSC.MouthLowerDown_R", "to": "Standard.MouthLowerDown_R" },
|
||||
{ "from": "OSC.MouthPress_L", "to": "Standard.MouthPress_L" },
|
||||
{ "from": "OSC.MouthPress_R", "to": "Standard.MouthPress_R" },
|
||||
{ "from": "OSC.MouthShrugLower", "to": "Standard.MouthShrugLower" },
|
||||
{ "from": "OSC.MouthShrugUpper", "to": "Standard.MouthShrugUpper" },
|
||||
{ "from": "OSC.NoseSneer_L", "to": "Standard.NoseSneer_L" },
|
||||
{ "from": "OSC.NoseSneer_R", "to": "Standard.NoseSneer_R" },
|
||||
{ "from": "OSC.TongueOut", "to": "Standard.TongueOut" },
|
||||
{ "from": "OSC.UserBlendshape0", "to": "Standard.UserBlendshape0" },
|
||||
{ "from": "OSC.UserBlendshape1", "to": "Standard.UserBlendshape1" },
|
||||
{ "from": "OSC.UserBlendshape2", "to": "Standard.UserBlendshape2" },
|
||||
{ "from": "OSC.UserBlendshape3", "to": "Standard.UserBlendshape3" },
|
||||
{ "from": "OSC.UserBlendshape4", "to": "Standard.UserBlendshape4" },
|
||||
{ "from": "OSC.UserBlendshape5", "to": "Standard.UserBlendshape5" },
|
||||
{ "from": "OSC.UserBlendshape6", "to": "Standard.UserBlendshape6" },
|
||||
{ "from": "OSC.UserBlendshape7", "to": "Standard.UserBlendshape7" },
|
||||
{ "from": "OSC.UserBlendshape8", "to": "Standard.UserBlendshape8" },
|
||||
{ "from": "OSC.UserBlendshape9", "to": "Standard.UserBlendshape9" }
|
||||
]
|
||||
}
|
||||
|
|
@ -332,7 +332,7 @@ Item {
|
|||
anchors.fill: stackView
|
||||
id: controllerPrefereneces
|
||||
objectName: "TabletControllerPreferences"
|
||||
showCategories: ["VR Movement", "Game Controller", "Sixense Controllers", "Perception Neuron", "Leap Motion"]
|
||||
showCategories: ["VR Movement", "Game Controller", "Sixense Controllers", "Perception Neuron", "Leap Motion", "Open Sound Control (OSC)"]
|
||||
categoryProperties: {
|
||||
"VR Movement" : {
|
||||
"User real-world height (meters)" : { "anchors.right" : "undefined" },
|
||||
|
|
|
|||
|
|
@ -6592,22 +6592,23 @@ void MyAvatar::updateEyesLookAtPosition(float deltaTime) {
|
|||
int rightEyeJointIndex = getJointIndex("RightEye");
|
||||
bool eyesAreOverridden = getIsJointOverridden(leftEyeJointIndex) ||
|
||||
getIsJointOverridden(rightEyeJointIndex);
|
||||
const float DEFAULT_GAZE_DISTANCE = 20.0f; // meters
|
||||
if (eyesAreOverridden) {
|
||||
// A script has set the eye rotations, so use these to set lookAtSpot
|
||||
glm::quat leftEyeRotation = getAbsoluteJointRotationInObjectFrame(leftEyeJointIndex);
|
||||
glm::quat rightEyeRotation = getAbsoluteJointRotationInObjectFrame(rightEyeJointIndex);
|
||||
glm::vec3 leftVec = getWorldOrientation() * leftEyeRotation * IDENTITY_FORWARD;
|
||||
glm::vec3 rightVec = getWorldOrientation() * rightEyeRotation * IDENTITY_FORWARD;
|
||||
glm::vec3 leftVec = getWorldOrientation() * leftEyeRotation * Vectors::UNIT_Z;
|
||||
glm::vec3 rightVec = getWorldOrientation() * rightEyeRotation * Vectors::UNIT_Z;
|
||||
glm::vec3 leftEyePosition = myHead->getLeftEyePosition();
|
||||
glm::vec3 rightEyePosition = myHead->getRightEyePosition();
|
||||
float t1, t2;
|
||||
bool success = findClosestApproachOfLines(leftEyePosition, leftVec, rightEyePosition, rightVec, t1, t2);
|
||||
if (success) {
|
||||
if (success && t1 > 0 && t2 > 0) {
|
||||
glm::vec3 leftFocus = leftEyePosition + leftVec * t1;
|
||||
glm::vec3 rightFocus = rightEyePosition + rightVec * t2;
|
||||
lookAtSpot = (leftFocus + rightFocus) / 2.0f; // average
|
||||
} else {
|
||||
lookAtSpot = myHead->getEyePosition() + glm::normalize(leftVec) * 1000.0f;
|
||||
lookAtSpot = myHead->getEyePosition() + glm::normalize(leftVec) * DEFAULT_GAZE_DISTANCE;
|
||||
}
|
||||
} else if (_scriptControlsEyesLookAt) {
|
||||
if (_scriptEyesControlTimer < MAX_LOOK_AT_TIME_SCRIPT_CONTROL) {
|
||||
|
|
@ -6621,19 +6622,18 @@ void MyAvatar::updateEyesLookAtPosition(float deltaTime) {
|
|||
controller::Pose rightEyePose = getControllerPoseInAvatarFrame(controller::Action::RIGHT_EYE);
|
||||
if (leftEyePose.isValid() && rightEyePose.isValid()) {
|
||||
// an eye tracker is in use, set lookAtSpot from this
|
||||
glm::vec3 leftVec = getWorldOrientation() * leftEyePose.rotation * glm::vec3(0.0f, 0.0f, -1.0f);
|
||||
glm::vec3 rightVec = getWorldOrientation() * rightEyePose.rotation * glm::vec3(0.0f, 0.0f, -1.0f);
|
||||
|
||||
glm::vec3 leftVec = getWorldOrientation() * leftEyePose.rotation * Vectors::UNIT_Z;
|
||||
glm::vec3 rightVec = getWorldOrientation() * rightEyePose.rotation * Vectors::UNIT_Z;
|
||||
glm::vec3 leftEyePosition = myHead->getLeftEyePosition();
|
||||
glm::vec3 rightEyePosition = myHead->getRightEyePosition();
|
||||
float t1, t2;
|
||||
bool success = findClosestApproachOfLines(leftEyePosition, leftVec, rightEyePosition, rightVec, t1, t2);
|
||||
if (success) {
|
||||
if (success && t1 > 0 && t2 > 0) {
|
||||
glm::vec3 leftFocus = leftEyePosition + leftVec * t1;
|
||||
glm::vec3 rightFocus = rightEyePosition + rightVec * t2;
|
||||
lookAtSpot = (leftFocus + rightFocus) / 2.0f; // average
|
||||
} else {
|
||||
lookAtSpot = myHead->getEyePosition() + glm::normalize(leftVec) * 1000.0f;
|
||||
lookAtSpot = myHead->getEyePosition() + glm::normalize(leftVec) * DEFAULT_GAZE_DISTANCE;
|
||||
}
|
||||
} else {
|
||||
// no script override, no eye tracker, so do procedural eye motion
|
||||
|
|
|
|||
|
|
@ -134,11 +134,11 @@ void MyHead::simulate(float deltaTime) {
|
|||
userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_L) ||
|
||||
userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_R);
|
||||
|
||||
bool eyesTracked =
|
||||
userInputMapper->getPoseState(controller::Action::LEFT_EYE).valid &&
|
||||
userInputMapper->getPoseState(controller::Action::RIGHT_EYE).valid;
|
||||
|
||||
MyAvatar* myAvatar = static_cast<MyAvatar*>(_owningAvatar);
|
||||
bool eyesTracked =
|
||||
myAvatar->getControllerPoseInSensorFrame(controller::Action::LEFT_EYE).valid &&
|
||||
myAvatar->getControllerPoseInSensorFrame(controller::Action::RIGHT_EYE).valid;
|
||||
|
||||
int leftEyeJointIndex = myAvatar->getJointIndex("LeftEye");
|
||||
int rightEyeJointIndex = myAvatar->getJointIndex("RightEye");
|
||||
bool eyeJointsOverridden = myAvatar->getIsJointOverridden(leftEyeJointIndex) || myAvatar->getIsJointOverridden(rightEyeJointIndex);
|
||||
|
|
|
|||
|
|
@ -96,17 +96,21 @@ void Head::simulate(float deltaTime) {
|
|||
const float BLINK_START_VARIABILITY = 0.25f;
|
||||
const float FULLY_OPEN = 0.0f;
|
||||
const float FULLY_CLOSED = 1.0f;
|
||||
const float TALKING_LOUDNESS = 150.0f;
|
||||
|
||||
_timeWithoutTalking += deltaTime;
|
||||
if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) {
|
||||
_timeWithoutTalking = 0.0f;
|
||||
}
|
||||
|
||||
if (getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) &&
|
||||
!getSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation)) {
|
||||
|
||||
// handle automatic blinks
|
||||
// Detect transition from talking to not; force blink after that and a delay
|
||||
bool forceBlink = false;
|
||||
const float TALKING_LOUDNESS = 150.0f;
|
||||
const float BLINK_AFTER_TALKING = 0.25f;
|
||||
_timeWithoutTalking += deltaTime;
|
||||
if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) {
|
||||
_timeWithoutTalking = 0.0f;
|
||||
} else if (_timeWithoutTalking - deltaTime < BLINK_AFTER_TALKING && _timeWithoutTalking >= BLINK_AFTER_TALKING) {
|
||||
if (_timeWithoutTalking - deltaTime < BLINK_AFTER_TALKING && _timeWithoutTalking >= BLINK_AFTER_TALKING) {
|
||||
forceBlink = true;
|
||||
}
|
||||
if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) {
|
||||
|
|
@ -150,11 +154,13 @@ void Head::simulate(float deltaTime) {
|
|||
} else {
|
||||
_rightEyeBlink = FULLY_OPEN;
|
||||
_leftEyeBlink = FULLY_OPEN;
|
||||
updateEyeLookAt();
|
||||
}
|
||||
|
||||
// use data to update fake Faceshift blendshape coefficients
|
||||
if (getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) &&
|
||||
!getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) {
|
||||
|
||||
// Update audio attack data for facial animation (eyebrows and mouth)
|
||||
float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz
|
||||
_audioAttack = audioAttackAveragingRate * _audioAttack +
|
||||
|
|
@ -188,6 +194,7 @@ void Head::simulate(float deltaTime) {
|
|||
|
||||
if (getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) &&
|
||||
!getSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation)) {
|
||||
|
||||
// This controls two things, the eye brow and the upper eye lid, it is driven by the vertical up/down angle of the
|
||||
// eyes relative to the head. This is to try to help prevent sleepy eyes/crazy eyes.
|
||||
applyEyelidOffset(getOrientation());
|
||||
|
|
|
|||
|
|
@ -376,7 +376,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent
|
|||
tranlationChangedSince(lastSentTime) ||
|
||||
parentInfoChangedSince(lastSentTime));
|
||||
hasHandControllers = _controllerLeftHandMatrixCache.isValid() || _controllerRightHandMatrixCache.isValid();
|
||||
hasFaceTrackerInfo = !dropFaceTracking && getHasScriptedBlendshapes() &&
|
||||
hasFaceTrackerInfo = !dropFaceTracking && (getHasScriptedBlendshapes() || _headData->_hasInputDrivenBlendshapes) &&
|
||||
(sendAll || faceTrackerInfoChangedSince(lastSentTime));
|
||||
hasJointData = !sendMinimum;
|
||||
hasJointDefaultPoseFlags = hasJointData;
|
||||
|
|
|
|||
|
|
@ -36,6 +36,12 @@ if (NOT SERVER_ONLY AND NOT ANDROID)
|
|||
add_subdirectory(${DIR})
|
||||
set(DIR "hifiLeapMotion")
|
||||
add_subdirectory(${DIR})
|
||||
|
||||
if (WIN32)
|
||||
set(DIR "hifiOsc")
|
||||
add_subdirectory(${DIR})
|
||||
endif()
|
||||
|
||||
endif()
|
||||
|
||||
# server-side plugins
|
||||
|
|
|
|||
14
plugins/hifiOsc/CMakeLists.txt
Normal file
14
plugins/hifiOsc/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
#
|
||||
# Created by Anthony Thibault on 2019/8/24
|
||||
# Copyright 2019 High Fidelity, Inc.
|
||||
#
|
||||
# Distributed under the Apache License, Version 2.0.
|
||||
# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
|
||||
#
|
||||
|
||||
set(TARGET_NAME hifiOsc)
|
||||
setup_hifi_plugin(Qml)
|
||||
link_hifi_libraries(shared controllers ui plugins input-plugins display-plugins)
|
||||
target_liblo()
|
||||
|
||||
|
||||
636
plugins/hifiOsc/src/OscPlugin.cpp
Normal file
636
plugins/hifiOsc/src/OscPlugin.cpp
Normal file
|
|
@ -0,0 +1,636 @@
|
|||
//
|
||||
// OscPlugin.cpp
|
||||
//
|
||||
// Created by Anthony Thibault on 2019/8/24
|
||||
// Copyright 2019 High Fidelity, Inc.
|
||||
//
|
||||
// Distributed under the Apache License, Version 2.0.
|
||||
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
//
|
||||
|
||||
#include "OscPlugin.h"
|
||||
|
||||
#include <controllers/UserInputMapper.h>
|
||||
#include <QLoggingCategory>
|
||||
#include <PathUtils.h>
|
||||
#include <DebugDraw.h>
|
||||
#include <cassert>
|
||||
#include <NumericalConstants.h>
|
||||
#include <StreamUtils.h>
|
||||
#include <Preferences.h>
|
||||
#include <SettingHandle.h>
|
||||
|
||||
Q_DECLARE_LOGGING_CATEGORY(inputplugins)
|
||||
Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins")
|
||||
|
||||
const char* OscPlugin::NAME = "Open Sound Control (OSC)";
|
||||
const char* OscPlugin::OSC_ID_STRING = "Open Sound Control (OSC)";
|
||||
const bool DEFAULT_ENABLED = false;
|
||||
|
||||
enum class FaceCap {
|
||||
BrowsU_C = 0,
|
||||
BrowsD_L,
|
||||
BrowsD_R,
|
||||
BrowsU_L,
|
||||
BrowsU_R,
|
||||
EyeUp_L,
|
||||
EyeUp_R,
|
||||
EyeDown_L,
|
||||
EyeDown_R,
|
||||
EyeIn_L,
|
||||
EyeIn_R,
|
||||
EyeOut_L,
|
||||
EyeOut_R,
|
||||
EyeBlink_L,
|
||||
EyeBlink_R,
|
||||
EyeSquint_L,
|
||||
EyeSquint_R,
|
||||
EyeOpen_L,
|
||||
EyeOpen_R,
|
||||
Puff,
|
||||
CheekSquint_L,
|
||||
CheekSquint_R,
|
||||
NoseSneer_L,
|
||||
NoseSneer_R,
|
||||
JawOpen,
|
||||
JawFwd,
|
||||
JawLeft,
|
||||
JawRight,
|
||||
LipsFunnel,
|
||||
LipsPucker,
|
||||
MouthLeft,
|
||||
MouthRight,
|
||||
LipsUpperClose,
|
||||
LipsLowerClose,
|
||||
MouthShrugUpper,
|
||||
MouthShrugLower,
|
||||
MouthClose,
|
||||
MouthSmile_L,
|
||||
MouthSmile_R,
|
||||
MouthFrown_L,
|
||||
MouthFrown_R,
|
||||
MouthDimple_L,
|
||||
MouthDimple_R,
|
||||
MouthUpperUp_L,
|
||||
MouthUpperUp_R,
|
||||
MouthLowerDown_L,
|
||||
MouthLowerDown_R,
|
||||
MouthPress_L,
|
||||
MouthPress_R,
|
||||
LipsStretch_L,
|
||||
LipsStretch_R,
|
||||
TongueOut,
|
||||
BlendshapeCount
|
||||
};
|
||||
|
||||
// used to mirror left/right shapes from FaceCap.
|
||||
// i.e. right and left shapes are swapped.
|
||||
FaceCap faceMirrorMap[(int)FaceCap::BlendshapeCount] = {
|
||||
FaceCap::BrowsU_C,
|
||||
FaceCap::BrowsD_R,
|
||||
FaceCap::BrowsD_L,
|
||||
FaceCap::BrowsU_R,
|
||||
FaceCap::BrowsU_L,
|
||||
FaceCap::EyeUp_R,
|
||||
FaceCap::EyeUp_L,
|
||||
FaceCap::EyeDown_R,
|
||||
FaceCap::EyeDown_L,
|
||||
FaceCap::EyeIn_R,
|
||||
FaceCap::EyeIn_L,
|
||||
FaceCap::EyeOut_R,
|
||||
FaceCap::EyeOut_L,
|
||||
FaceCap::EyeBlink_R,
|
||||
FaceCap::EyeBlink_L,
|
||||
FaceCap::EyeSquint_R,
|
||||
FaceCap::EyeSquint_L,
|
||||
FaceCap::EyeOpen_R,
|
||||
FaceCap::EyeOpen_L,
|
||||
FaceCap::Puff,
|
||||
FaceCap::CheekSquint_R,
|
||||
FaceCap::CheekSquint_L,
|
||||
FaceCap::NoseSneer_R,
|
||||
FaceCap::NoseSneer_L,
|
||||
FaceCap::JawOpen,
|
||||
FaceCap::JawFwd,
|
||||
FaceCap::JawRight,
|
||||
FaceCap::JawLeft,
|
||||
FaceCap::LipsFunnel,
|
||||
FaceCap::LipsPucker,
|
||||
FaceCap::MouthRight,
|
||||
FaceCap::MouthLeft,
|
||||
FaceCap::LipsUpperClose,
|
||||
FaceCap::LipsLowerClose,
|
||||
FaceCap::MouthShrugUpper,
|
||||
FaceCap::MouthShrugLower,
|
||||
FaceCap::MouthClose,
|
||||
FaceCap::MouthSmile_R,
|
||||
FaceCap::MouthSmile_L,
|
||||
FaceCap::MouthFrown_R,
|
||||
FaceCap::MouthFrown_L,
|
||||
FaceCap::MouthDimple_R,
|
||||
FaceCap::MouthDimple_L,
|
||||
FaceCap::MouthUpperUp_R,
|
||||
FaceCap::MouthUpperUp_L,
|
||||
FaceCap::MouthLowerDown_R,
|
||||
FaceCap::MouthLowerDown_L,
|
||||
FaceCap::MouthPress_R,
|
||||
FaceCap::MouthPress_L,
|
||||
FaceCap::LipsStretch_R,
|
||||
FaceCap::LipsStretch_L,
|
||||
FaceCap::TongueOut
|
||||
};
|
||||
|
||||
static const char* STRINGS[FaceCap::BlendshapeCount] = {
|
||||
"BrowsU_C",
|
||||
"BrowsD_L",
|
||||
"BrowsD_R",
|
||||
"BrowsU_L",
|
||||
"BrowsU_R",
|
||||
"EyeUp_L",
|
||||
"EyeUp_R",
|
||||
"EyeDown_L",
|
||||
"EyeDown_R",
|
||||
"EyeIn_L",
|
||||
"EyeIn_R",
|
||||
"EyeOut_L",
|
||||
"EyeOut_R",
|
||||
"EyeBlink_L",
|
||||
"EyeBlink_R",
|
||||
"EyeSquint_L",
|
||||
"EyeSquint_R",
|
||||
"EyeOpen_L",
|
||||
"EyeOpen_R",
|
||||
"Puff",
|
||||
"CheekSquint_L",
|
||||
"CheekSquint_R",
|
||||
"NoseSneer_L",
|
||||
"NoseSneer_R",
|
||||
"JawOpen",
|
||||
"JawFwd",
|
||||
"JawLeft",
|
||||
"JawRight",
|
||||
"LipsFunnel",
|
||||
"LipsPucker",
|
||||
"MouthLeft",
|
||||
"MouthRight",
|
||||
"LipsUpperClose",
|
||||
"LipsLowerClose",
|
||||
"MouthShrugUpper",
|
||||
"MouthShrugLower",
|
||||
"MouthClose",
|
||||
"MouthSmile_L",
|
||||
"MouthSmile_R",
|
||||
"MouthFrown_L",
|
||||
"MouthFrown_R",
|
||||
"MouthDimple_L",
|
||||
"MouthDimple_R",
|
||||
"MouthUpperUp_L",
|
||||
"MouthUpperUp_R",
|
||||
"MouthLowerDown_L",
|
||||
"MouthLowerDown_R",
|
||||
"MouthPress_L",
|
||||
"MouthPress_R",
|
||||
"LipsStretch_L",
|
||||
"LipsStretch_R",
|
||||
"TongueOut"
|
||||
};
|
||||
|
||||
static enum controller::StandardAxisChannel CHANNELS[FaceCap::BlendshapeCount] = {
|
||||
controller::BROWSU_C,
|
||||
controller::BROWSD_L,
|
||||
controller::BROWSD_R,
|
||||
controller::BROWSU_L,
|
||||
controller::BROWSU_R,
|
||||
controller::EYEUP_L,
|
||||
controller::EYEUP_R,
|
||||
controller::EYEDOWN_L,
|
||||
controller::EYEDOWN_R,
|
||||
controller::EYEIN_L,
|
||||
controller::EYEIN_R,
|
||||
controller::EYEOUT_L,
|
||||
controller::EYEOUT_R,
|
||||
controller::EYEBLINK_L,
|
||||
controller::EYEBLINK_R,
|
||||
controller::EYESQUINT_L,
|
||||
controller::EYESQUINT_R,
|
||||
controller::EYEOPEN_L,
|
||||
controller::EYEOPEN_R,
|
||||
controller::PUFF,
|
||||
controller::CHEEKSQUINT_L,
|
||||
controller::CHEEKSQUINT_R,
|
||||
controller::NOSESNEER_L,
|
||||
controller::NOSESNEER_R,
|
||||
controller::JAWOPEN,
|
||||
controller::JAWFWD,
|
||||
controller::JAWLEFT,
|
||||
controller::JAWRIGHT,
|
||||
controller::LIPSFUNNEL,
|
||||
controller::LIPSPUCKER,
|
||||
controller::MOUTHLEFT,
|
||||
controller::MOUTHRIGHT,
|
||||
controller::LIPSUPPERCLOSE,
|
||||
controller::LIPSLOWERCLOSE,
|
||||
controller::MOUTHSHRUGUPPER,
|
||||
controller::MOUTHSHRUGLOWER,
|
||||
controller::MOUTHCLOSE,
|
||||
controller::MOUTHSMILE_L,
|
||||
controller::MOUTHSMILE_R,
|
||||
controller::MOUTHFROWN_L,
|
||||
controller::MOUTHFROWN_R,
|
||||
controller::MOUTHDIMPLE_L,
|
||||
controller::MOUTHDIMPLE_R,
|
||||
controller::MOUTHUPPERUP_L,
|
||||
controller::MOUTHUPPERUP_R,
|
||||
controller::MOUTHLOWERDOWN_L,
|
||||
controller::MOUTHLOWERDOWN_R,
|
||||
controller::MOUTHPRESS_L,
|
||||
controller::MOUTHPRESS_R,
|
||||
controller::LIPSSTRETCH_L,
|
||||
controller::LIPSSTRETCH_R,
|
||||
controller::TONGUEOUT
|
||||
};
|
||||
|
||||
|
||||
void OscPlugin::init() {
|
||||
|
||||
_inputDevice = std::make_shared<InputDevice>();
|
||||
_inputDevice->setContainer(this);
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> guard(_dataMutex);
|
||||
_blendshapeValues.assign((int)FaceCap::BlendshapeCount, 0.0f);
|
||||
_blendshapeValidFlags.assign((int)FaceCap::BlendshapeCount, false);
|
||||
_headRot = glm::quat();
|
||||
_headRotValid = false;
|
||||
_headTransTarget = extractTranslation(_lastInputCalibrationData.defaultHeadMat);
|
||||
_headTransSmoothed = extractTranslation(_lastInputCalibrationData.defaultHeadMat);
|
||||
_headTransValid = false;
|
||||
_eyeLeftRot = glm::quat();
|
||||
_eyeLeftRotValid = false;
|
||||
_eyeRightRot = glm::quat();
|
||||
_eyeRightRotValid = false;
|
||||
}
|
||||
|
||||
loadSettings();
|
||||
|
||||
auto preferences = DependencyManager::get<Preferences>();
|
||||
static const QString OSC_PLUGIN { OscPlugin::NAME };
|
||||
{
|
||||
auto getter = [this]()->bool { return _enabled; };
|
||||
auto setter = [this](bool value) {
|
||||
_enabled = value;
|
||||
saveSettings();
|
||||
if (!_enabled) {
|
||||
auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
|
||||
userInputMapper->withLock([&, this]() {
|
||||
_inputDevice->clearState();
|
||||
restartServer();
|
||||
});
|
||||
}
|
||||
};
|
||||
auto preference = new CheckPreference(OSC_PLUGIN, "Enabled", getter, setter);
|
||||
preferences->addPreference(preference);
|
||||
}
|
||||
{
|
||||
auto debugGetter = [this]()->bool { return _debug; };
|
||||
auto debugSetter = [this](bool value) {
|
||||
_debug = value;
|
||||
saveSettings();
|
||||
};
|
||||
auto preference = new CheckPreference(OSC_PLUGIN, "Extra Debugging", debugGetter, debugSetter);
|
||||
preferences->addPreference(preference);
|
||||
}
|
||||
|
||||
{
|
||||
static const int MIN_PORT_NUMBER { 0 };
|
||||
static const int MAX_PORT_NUMBER { 65535 };
|
||||
|
||||
auto getter = [this]()->float { return (float)_serverPort; };
|
||||
auto setter = [this](float value) {
|
||||
_serverPort = (int)value;
|
||||
saveSettings();
|
||||
restartServer();
|
||||
};
|
||||
auto preference = new SpinnerPreference(OSC_PLUGIN, "Server Port", getter, setter);
|
||||
|
||||
preference->setMin(MIN_PORT_NUMBER);
|
||||
preference->setMax(MAX_PORT_NUMBER);
|
||||
preference->setStep(1);
|
||||
preferences->addPreference(preference);
|
||||
}
|
||||
}
|
||||
|
||||
bool OscPlugin::isSupported() const {
|
||||
// networking/UDP is pretty much always available...
|
||||
return true;
|
||||
}
|
||||
|
||||
static void errorHandlerFunc(int num, const char* msg, const char* path) {
|
||||
qDebug(inputplugins) << "OscPlugin: server error" << num << "in path" << path << ":" << msg;
|
||||
}
|
||||
|
||||
static int genericHandlerFunc(const char* path, const char* types, lo_arg** argv,
|
||||
int argc, void* data, void* user_data) {
|
||||
|
||||
OscPlugin* container = reinterpret_cast<OscPlugin*>(user_data);
|
||||
assert(container);
|
||||
|
||||
QString key(path);
|
||||
std::lock_guard<std::mutex> guard(container->_dataMutex);
|
||||
|
||||
// Special case: decode blendshapes from face-cap iPhone app.
|
||||
// http://www.bannaflak.com/face-cap/livemode.html
|
||||
if (path[0] == '/' && path[1] == 'W' && argc == 2 && types[0] == 'i' && types[1] == 'f') {
|
||||
int index = argv[0]->i;
|
||||
if (index >= 0 && index < (int)FaceCap::BlendshapeCount) {
|
||||
int mirroredIndex = (int)faceMirrorMap[index];
|
||||
container->_blendshapeValues[mirroredIndex] = argv[1]->f;
|
||||
container->_blendshapeValidFlags[mirroredIndex] = true;
|
||||
}
|
||||
}
|
||||
|
||||
// map /HT to head translation
|
||||
if (path[0] == '/' && path[1] == 'H' && path[2] == 'T' &&
|
||||
types[0] == 'f' && types[1] == 'f' && types[2] == 'f') {
|
||||
glm::vec3 trans(-argv[0]->f, -argv[1]->f, argv[2]->f); // in cm
|
||||
|
||||
// convert trans into a delta (in meters) from the sweet spot of the iphone camera.
|
||||
const float CM_TO_METERS = 0.01f;
|
||||
const glm::vec3 FACE_CAP_HEAD_SWEET_SPOT(0.0f, 0.0f, -45.0f);
|
||||
glm::vec3 delta = (trans - FACE_CAP_HEAD_SWEET_SPOT) * CM_TO_METERS;
|
||||
|
||||
container->_headTransTarget = extractTranslation(container->_lastInputCalibrationData.defaultHeadMat) + delta;
|
||||
container->_headTransValid = true;
|
||||
}
|
||||
|
||||
// map /HR to head rotation
|
||||
if (path[0] == '/' && path[1] == 'H' && path[2] == 'R' && path[3] == 0 &&
|
||||
types[0] == 'f' && types[1] == 'f' && types[2] == 'f') {
|
||||
glm::vec3 euler(-argv[0]->f, -argv[1]->f, argv[2]->f);
|
||||
container->_headRot = glm::quat(glm::radians(euler)) * Quaternions::Y_180;
|
||||
container->_headRotValid = true;
|
||||
}
|
||||
|
||||
// map /ELR to left eye rot
|
||||
if (path[0] == '/' && path[1] == 'E' && path[2] == 'L' && path[3] == 'R' &&
|
||||
types[0] == 'f' && types[1] == 'f') {
|
||||
glm::vec3 euler(argv[0]->f, -argv[1]->f, 0.0f);
|
||||
container->_eyeLeftRot = container->_headRot * glm::quat(glm::radians(euler));
|
||||
container->_eyeLeftRotValid = true;
|
||||
}
|
||||
|
||||
// map /ERR to right eye rot
|
||||
if (path[0] == '/' && path[1] == 'E' && path[2] == 'R' && path[3] == 'R' &&
|
||||
types[0] == 'f' && types[1] == 'f') {
|
||||
glm::vec3 euler((float)argv[0]->f, (float)-argv[1]->f, 0.0f);
|
||||
container->_eyeRightRot = container->_headRot * glm::quat(glm::radians(euler));
|
||||
container->_eyeRightRotValid = true;
|
||||
}
|
||||
|
||||
// AJT: TODO map /STRINGS[i] to _blendshapeValues[i]
|
||||
|
||||
if (container->_debug) {
|
||||
for (int i = 0; i < argc; i++) {
|
||||
switch (types[i]) {
|
||||
case 'i':
|
||||
// int32
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->i;
|
||||
break;
|
||||
case 'f':
|
||||
// float32
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->f32;
|
||||
break;
|
||||
case 's':
|
||||
// OSC-string
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <string>";
|
||||
break;
|
||||
case 'b':
|
||||
// OSC-blob
|
||||
break;
|
||||
case 'h':
|
||||
// 64 bit big-endian two's complement integer
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->h;
|
||||
break;
|
||||
case 't':
|
||||
// OSC-timetag
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <OSC-timetag>";
|
||||
break;
|
||||
case 'd':
|
||||
// 64 bit ("double") IEEE 754 floating point number
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->d;
|
||||
break;
|
||||
case 'S':
|
||||
// Alternate type represented as an OSC-string (for example, for systems that differentiate "symbols" from "strings")
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <OSC-symbol>";
|
||||
break;
|
||||
case 'c':
|
||||
// an ascii character, sent as 32 bits
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->c;
|
||||
break;
|
||||
case 'r':
|
||||
// 32 bit RGBA color
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <color>";
|
||||
break;
|
||||
case 'm':
|
||||
// 4 byte MIDI message. Bytes from MSB to LSB are: port id, status byte, data1, data2
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <midi>";
|
||||
break;
|
||||
case 'T':
|
||||
// true
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <true>";
|
||||
break;
|
||||
case 'F':
|
||||
// false
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <false>";
|
||||
break;
|
||||
case 'N':
|
||||
// nil
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <nil>";
|
||||
break;
|
||||
case 'I':
|
||||
// inf
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <inf>";
|
||||
break;
|
||||
case '[':
|
||||
// Indicates the beginning of an array. The tags following are for data in the Array until a close brace tag is reached.
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <begin-array>";
|
||||
break;
|
||||
case ']':
|
||||
// Indicates the end of an array.
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <end-array>";
|
||||
break;
|
||||
default:
|
||||
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <unknown-type>" << types[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
bool OscPlugin::activate() {
|
||||
InputPlugin::activate();
|
||||
|
||||
loadSettings();
|
||||
|
||||
if (_enabled) {
|
||||
|
||||
qDebug(inputplugins) << "OscPlugin: activated";
|
||||
|
||||
// register with userInputMapper
|
||||
auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
|
||||
userInputMapper->registerDevice(_inputDevice);
|
||||
|
||||
return startServer();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void OscPlugin::deactivate() {
|
||||
qDebug(inputplugins) << "OscPlugin: deactivated, _oscServerThread =" << _oscServerThread;
|
||||
|
||||
if (_oscServerThread) {
|
||||
stopServer();
|
||||
}
|
||||
}
|
||||
|
||||
void OscPlugin::pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
|
||||
if (!_enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
_lastInputCalibrationData = inputCalibrationData;
|
||||
|
||||
auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
|
||||
userInputMapper->withLock([&, this]() {
|
||||
_inputDevice->update(deltaTime, inputCalibrationData);
|
||||
});
|
||||
}
|
||||
|
||||
void OscPlugin::saveSettings() const {
|
||||
Settings settings;
|
||||
QString idString = getID();
|
||||
settings.beginGroup(idString);
|
||||
{
|
||||
settings.setValue(QString("enabled"), _enabled);
|
||||
settings.setValue(QString("extraDebug"), _debug);
|
||||
settings.setValue(QString("serverPort"), _serverPort);
|
||||
}
|
||||
settings.endGroup();
|
||||
}
|
||||
|
||||
void OscPlugin::loadSettings() {
|
||||
Settings settings;
|
||||
QString idString = getID();
|
||||
settings.beginGroup(idString);
|
||||
{
|
||||
_enabled = settings.value("enabled", QVariant(DEFAULT_ENABLED)).toBool();
|
||||
_debug = settings.value("extraDebug", QVariant(DEFAULT_ENABLED)).toBool();
|
||||
_serverPort = settings.value("serverPort", QVariant(DEFAULT_OSC_SERVER_PORT)).toInt();
|
||||
}
|
||||
settings.endGroup();
|
||||
}
|
||||
|
||||
bool OscPlugin::startServer() {
|
||||
if (_oscServerThread) {
|
||||
qWarning(inputplugins) << "OscPlugin: (startServer) server is already running, _oscServerThread =" << _oscServerThread;
|
||||
return true;
|
||||
}
|
||||
|
||||
// start a new server on specified port
|
||||
const size_t BUFFER_SIZE = 64;
|
||||
char serverPortString[BUFFER_SIZE];
|
||||
snprintf(serverPortString, BUFFER_SIZE, "%d", _serverPort);
|
||||
_oscServerThread = lo_server_thread_new(serverPortString, errorHandlerFunc);
|
||||
|
||||
qDebug(inputplugins) << "OscPlugin: server started on port" << serverPortString << ", _oscServerThread =" << _oscServerThread;
|
||||
|
||||
// add method that will match any path and args
|
||||
// NOTE: callback function will be called on the OSC thread, not the appliation thread.
|
||||
lo_server_thread_add_method(_oscServerThread, NULL, NULL, genericHandlerFunc, (void*)this);
|
||||
|
||||
lo_server_thread_start(_oscServerThread);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void OscPlugin::stopServer() {
|
||||
if (!_oscServerThread) {
|
||||
qWarning(inputplugins) << "OscPlugin: (stopServer) server is already shutdown.";
|
||||
}
|
||||
|
||||
// stop and free server
|
||||
lo_server_thread_stop(_oscServerThread);
|
||||
lo_server_thread_free(_oscServerThread);
|
||||
_oscServerThread = nullptr;
|
||||
}
|
||||
|
||||
void OscPlugin::restartServer() {
|
||||
if (_oscServerThread) {
|
||||
stopServer();
|
||||
}
|
||||
startServer();
|
||||
}
|
||||
|
||||
//
|
||||
// InputDevice
|
||||
//
|
||||
|
||||
controller::Input::NamedVector OscPlugin::InputDevice::getAvailableInputs() const {
|
||||
static controller::Input::NamedVector availableInputs;
|
||||
if (availableInputs.size() == 0) {
|
||||
for (int i = 0; i < (int)FaceCap::BlendshapeCount; i++) {
|
||||
availableInputs.push_back(makePair(CHANNELS[i], STRINGS[i]));
|
||||
}
|
||||
}
|
||||
availableInputs.push_back(makePair(controller::HEAD, "Head"));
|
||||
availableInputs.push_back(makePair(controller::LEFT_EYE, "LeftEye"));
|
||||
availableInputs.push_back(makePair(controller::RIGHT_EYE, "RightEye"));
|
||||
return availableInputs;
|
||||
}
|
||||
|
||||
QString OscPlugin::InputDevice::getDefaultMappingConfig() const {
|
||||
static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/osc.json";
|
||||
return MAPPING_JSON;
|
||||
}
|
||||
|
||||
void OscPlugin::InputDevice::update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
|
||||
glm::mat4 sensorToAvatarMat = glm::inverse(inputCalibrationData.avatarMat) * inputCalibrationData.sensorToWorldMat;
|
||||
std::lock_guard<std::mutex> guard(_container->_dataMutex);
|
||||
for (int i = 0; i < (int)FaceCap::BlendshapeCount; i++) {
|
||||
if (_container->_blendshapeValidFlags[i]) {
|
||||
_axisStateMap[CHANNELS[i]] = controller::AxisValue(_container->_blendshapeValues[i], 0, true);
|
||||
}
|
||||
}
|
||||
if (_container->_headRotValid && _container->_headTransValid) {
|
||||
const float SMOOTH_TIMESCALE = 2.0f;
|
||||
float tau = deltaTime / SMOOTH_TIMESCALE;
|
||||
_container->_headTransSmoothed = lerp(_container->_headTransSmoothed, _container->_headTransTarget, tau);
|
||||
glm::vec3 delta = _container->_headTransSmoothed - _container->_headTransTarget;
|
||||
glm::vec3 trans = extractTranslation(inputCalibrationData.defaultHeadMat) + delta;
|
||||
|
||||
controller::Pose sensorSpacePose(trans, _container->_headRot);
|
||||
_poseStateMap[controller::HEAD] = sensorSpacePose.transform(sensorToAvatarMat);
|
||||
}
|
||||
if (_container->_eyeLeftRotValid) {
|
||||
controller::Pose sensorSpacePose(vec3(0.0f), _container->_eyeLeftRot);
|
||||
_poseStateMap[controller::LEFT_EYE] = sensorSpacePose.transform(sensorToAvatarMat);
|
||||
}
|
||||
if (_container->_eyeRightRotValid) {
|
||||
controller::Pose sensorSpacePose(vec3(0.0f), _container->_eyeRightRot);
|
||||
_poseStateMap[controller::RIGHT_EYE] = sensorSpacePose.transform(sensorToAvatarMat);
|
||||
}
|
||||
}
|
||||
|
||||
void OscPlugin::InputDevice::clearState() {
|
||||
std::lock_guard<std::mutex> guard(_container->_dataMutex);
|
||||
for (int i = 0; i < (int)FaceCap::BlendshapeCount; i++) {
|
||||
_axisStateMap[CHANNELS[i]] = controller::AxisValue(0.0f, 0, false);
|
||||
}
|
||||
_poseStateMap[controller::HEAD] = controller::Pose();
|
||||
_poseStateMap[controller::LEFT_EYE] = controller::Pose();
|
||||
_poseStateMap[controller::RIGHT_EYE] = controller::Pose();
|
||||
}
|
||||
|
||||
95
plugins/hifiOsc/src/OscPlugin.h
Normal file
95
plugins/hifiOsc/src/OscPlugin.h
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
//
|
||||
// OscPlugin.h
|
||||
//
|
||||
// Created by Anthony Thibault on 2019/8/24
|
||||
// Copyright 2019 High Fidelity, Inc.
|
||||
//
|
||||
// Distributed under the Apache License, Version 2.0.
|
||||
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
//
|
||||
|
||||
#ifndef hifi_OscPlugin_h
|
||||
#define hifi_OscPlugin_h
|
||||
|
||||
#include <controllers/InputDevice.h>
|
||||
#include <controllers/StandardControls.h>
|
||||
#include <plugins/InputPlugin.h>
|
||||
|
||||
#include "lo/lo.h"
|
||||
|
||||
// OSC (Open Sound Control) input plugin.
|
||||
class OscPlugin : public InputPlugin {
|
||||
Q_OBJECT
|
||||
public:
|
||||
|
||||
// Plugin functions
|
||||
virtual void init() override;
|
||||
virtual bool isSupported() const override;
|
||||
virtual const QString getName() const override { return NAME; }
|
||||
const QString getID() const override { return OSC_ID_STRING; }
|
||||
|
||||
virtual bool activate() override;
|
||||
virtual void deactivate() override;
|
||||
|
||||
virtual void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); }
|
||||
virtual void pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override;
|
||||
|
||||
virtual void saveSettings() const override;
|
||||
virtual void loadSettings() override;
|
||||
|
||||
bool startServer();
|
||||
void stopServer();
|
||||
void restartServer();
|
||||
|
||||
protected:
|
||||
|
||||
class InputDevice : public controller::InputDevice {
|
||||
public:
|
||||
friend class OscPlugin;
|
||||
|
||||
InputDevice() : controller::InputDevice("OSC") {}
|
||||
|
||||
// Device functions
|
||||
virtual controller::Input::NamedVector getAvailableInputs() const override;
|
||||
virtual QString getDefaultMappingConfig() const override;
|
||||
virtual void update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override;
|
||||
virtual void focusOutEvent() override {};
|
||||
|
||||
void clearState();
|
||||
void setContainer(OscPlugin* container) { _container = container; }
|
||||
|
||||
OscPlugin* _container { nullptr };
|
||||
};
|
||||
|
||||
std::shared_ptr<InputDevice> _inputDevice;
|
||||
|
||||
static const char* NAME;
|
||||
static const char* OSC_ID_STRING;
|
||||
|
||||
bool _enabled { false };
|
||||
mutable bool _initialized { false };
|
||||
|
||||
lo_server_thread _oscServerThread { nullptr };
|
||||
|
||||
public:
|
||||
bool _debug { false };
|
||||
enum Constants { DEFAULT_OSC_SERVER_PORT = 7700 };
|
||||
int _serverPort { DEFAULT_OSC_SERVER_PORT };
|
||||
controller::InputCalibrationData _lastInputCalibrationData;
|
||||
|
||||
std::vector<float> _blendshapeValues;
|
||||
std::vector<bool> _blendshapeValidFlags;
|
||||
glm::quat _headRot;
|
||||
bool _headRotValid { false };
|
||||
glm::vec3 _headTransTarget;
|
||||
glm::vec3 _headTransSmoothed;
|
||||
bool _headTransValid { false };
|
||||
glm::quat _eyeLeftRot;
|
||||
bool _eyeLeftRotValid { false };
|
||||
glm::quat _eyeRightRot;
|
||||
bool _eyeRightRotValid { false };
|
||||
std::mutex _dataMutex;
|
||||
};
|
||||
|
||||
#endif // hifi_OscPlugin_h
|
||||
|
||||
49
plugins/hifiOsc/src/OscProvider.cpp
Normal file
49
plugins/hifiOsc/src/OscProvider.cpp
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
//
|
||||
// Created by Anthony Thibault on 2019/8/25
|
||||
// Copyright 2019 High Fidelity, Inc.
|
||||
//
|
||||
// Distributed under the Apache License, Version 2.0.
|
||||
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
//
|
||||
|
||||
#include <mutex>
|
||||
|
||||
#include <QtCore/QObject>
|
||||
#include <QtCore/QtPlugin>
|
||||
#include <QtCore/QStringList>
|
||||
|
||||
#include <plugins/RuntimePlugin.h>
|
||||
#include <plugins/InputPlugin.h>
|
||||
|
||||
#include "OscPlugin.h"
|
||||
|
||||
class OscProvider : public QObject, public InputProvider
|
||||
{
|
||||
Q_OBJECT
|
||||
Q_PLUGIN_METADATA(IID InputProvider_iid FILE "plugin.json")
|
||||
Q_INTERFACES(InputProvider)
|
||||
|
||||
public:
|
||||
OscProvider(QObject* parent = nullptr) : QObject(parent) {}
|
||||
virtual ~OscProvider() {}
|
||||
|
||||
virtual InputPluginList getInputPlugins() override {
|
||||
static std::once_flag once;
|
||||
std::call_once(once, [&] {
|
||||
InputPluginPointer plugin(new OscPlugin());
|
||||
if (plugin->isSupported()) {
|
||||
_inputPlugins.push_back(plugin);
|
||||
}
|
||||
});
|
||||
return _inputPlugins;
|
||||
}
|
||||
|
||||
virtual void destroyInputPlugins() override {
|
||||
_inputPlugins.clear();
|
||||
}
|
||||
|
||||
private:
|
||||
InputPluginList _inputPlugins;
|
||||
};
|
||||
|
||||
#include "OscProvider.moc"
|
||||
4
plugins/hifiOsc/src/plugin.json
Normal file
4
plugins/hifiOsc/src/plugin.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"name":"Osc",
|
||||
"version": 1
|
||||
}
|
||||
Loading…
Reference in a new issue