Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-04-05 20:36:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Upgraded OSC plugin with changes from ARKit blendshape work.

Browse source
This commit is contained in:
Anthony J. Thibault 2019-11-19 13:10:47 -08:00
parent 6169936c2b
commit befbdcef48
12 changed files with 906 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmake/macros/TargetLiblo.cmake Normal file
View file

@ -0,0 +1,4 @@
macro(target_liblo)
find_library(LIBLO LIBLO)
target_link_libraries(${TARGET_NAME} ${LIBLO})
endmacro()

2
cmake/ports/hifi-client-deps/CONTROL
View file

@ -1,4 +1,4 @@
Source: hifi-client-deps
Version: 0
Description: Collected dependencies for High Fidelity applications
Build-Depends: hifi-deps, glslang, nlohmann-json, openvr (windows), quazip (!android), sdl2 (!android), spirv-cross (!android), spirv-tools (!android), vulkanmemoryallocator
Build-Depends: hifi-deps, glslang, nlohmann-json, openvr (windows), quazip (!android), sdl2 (!android), spirv-cross (!android), spirv-tools (!android), vulkanmemoryallocator, liblo (windows)

3
cmake/ports/liblo/CONTROL Normal file
View file

@ -0,0 +1,3 @@
Source: liblo
Version: 0.30
Description: liblo is an implementation of the Open Sound Control protocol for POSIX systems

36
cmake/ports/liblo/portfile.cmake Normal file
View file

@ -0,0 +1,36 @@
include(vcpkg_common_functions)
vcpkg_from_github(
OUT_SOURCE_PATH SOURCE_PATH
REPO radarsat1/liblo
REF 0.30
SHA512 d36c141c513f869e6d1963bd0d584030038019b8be0b27bb9a684722b6e7a38e942ad2ee7c2e67ac13b965560937aad97259435ed86034aa2dc8cb92d23845d8
HEAD_REF master
)
vcpkg_configure_cmake(
SOURCE_PATH ${SOURCE_PATH}/cmake
PREFER_NINJA # Disable this option if project cannot be built with Ninja
OPTIONS -DTHREADING=1
)
vcpkg_install_cmake()
# Install needed files into package directory
vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/liblo)
file(INSTALL ${CURRENT_PACKAGES_DIR}/bin/oscsend.exe DESTINATION ${CURRENT_PACKAGES_DIR}/tools/liblo)
file(INSTALL ${CURRENT_PACKAGES_DIR}/bin/oscdump.exe DESTINATION ${CURRENT_PACKAGES_DIR}/tools/liblo)
vcpkg_copy_tool_dependencies(${CURRENT_PACKAGES_DIR}/tools/liblo)
# Remove unnecessary files
file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
file(REMOVE ${CURRENT_PACKAGES_DIR}/bin/oscsend.exe ${CURRENT_PACKAGES_DIR}/bin/oscdump.exe)
file(REMOVE ${CURRENT_PACKAGES_DIR}/debug/bin/oscsend.exe ${CURRENT_PACKAGES_DIR}/debug/bin/oscdump.exe)
if(VCPKG_LIBRARY_LINKAGE STREQUAL static)
file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/bin ${CURRENT_PACKAGES_DIR}/debug/bin)
endif()
# Handle copyright
file(INSTALL ${SOURCE_PATH}/COPYING DESTINATION ${CURRENT_PACKAGES_DIR}/share/liblo RENAME copyright)

60
interface/resources/controllers/osc.json Normal file
View file

@ -0,0 +1,60 @@
{
"name": "OSC to Standard",
"channels": [
{ "from": "OSC.Head", "to" : "Standard.Head" },
{ "from": "OSC.LeftEye", "to" : "Standard.LeftEye" },
{ "from": "OSC.RightEye", "to" : "Standard.RightEye" },
{ "from": "OSC.BrowInnerUp", "to": "Standard.BrowInnerUp" },
{ "from": "OSC.BrowDownLeft", "to": "Standard.BrowDownLeft" },
{ "from": "OSC.BrowDownRight", "to": "Standard.BrowDownRight" },
{ "from": "OSC.BrowOuterUpLeft", "to": "Standard.BrowOuterUpLeft" },
{ "from": "OSC.BrowOuterUpRight", "to": "Standard.BrowOuterUpRight" },
{ "from": "OSC.EyeLookUpLeft", "to": "Standard.EyeLookUpLeft" },
{ "from": "OSC.EyeLookUpRight", "to": "Standard.EyeLookUpRight" },
{ "from": "OSC.EyeLookDownLeft", "to": "Standard.EyeLookDownLeft" },
{ "from": "OSC.EyeLookDownRight", "to": "Standard.EyeLookDownRight" },
{ "from": "OSC.EyeLookInLeft", "to": "Standard.EyeLookInLeft" },
{ "from": "OSC.EyeLookInRight", "to": "Standard.EyeLookInRight" },
{ "from": "OSC.EyeLookOutLeft", "to": "Standard.EyeLookOutLeft" },
{ "from": "OSC.EyeLookOutRight", "to": "Standard.EyeLookOutRight" },
{ "from": "OSC.EyeBlinkLeft", "to": "Standard.EyeBlinkLeft" },
{ "from": "OSC.EyeBlinkRight", "to": "Standard.EyeBlinkRight" },
{ "from": "OSC.EyeSquintLeft", "to": "Standard.EyeSquintLeft" },
{ "from": "OSC.EyeSquintRight", "to": "Standard.EyeSquintRight" },
{ "from": "OSC.EyeWideLeft", "to": "Standard.EyeWideLeft" },
{ "from": "OSC.EyeWideRight", "to": "Standard.EyeWideRight" },
{ "from": "OSC.CheekPuff", "to": "Standard.CheekPuff" },
{ "from": "OSC.CheekSquintLeft", "to": "Standard.CheekSquintLeft" },
{ "from": "OSC.CheekSquintRight", "to": "Standard.CheekSquintRight" },
{ "from": "OSC.NoseSneerLeft", "to": "Standard.NoseSneerLeft" },
{ "from": "OSC.NoseSneerRight", "to": "Standard.NoseSneerRight" },
{ "from": "OSC.JawOpen", "to": "Standard.JawOpen" },
{ "from": "OSC.JawForward", "to": "Standard.JawForward" },
{ "from": "OSC.JawLeft", "to": "Standard.JawLeft" },
{ "from": "OSC.JawRight", "to": "Standard.JawRight" },
{ "from": "OSC.MouthFunnel", "to": "Standard.MouthFunnel" },
{ "from": "OSC.MouthPucker", "to": "Standard.MouthPucker" },
{ "from": "OSC.MouthLeft", "to": "Standard.MouthLeft" },
{ "from": "OSC.MouthRight", "to": "Standard.MouthRight" },
{ "from": "OSC.MouthRollUpper", "to": "Standard.MouthRollUpper" },
{ "from": "OSC.MouthRollLower", "to": "Standard.MouthRollLower" },
{ "from": "OSC.MouthShrugUpper", "to": "Standard.MouthShrugUpper" },
{ "from": "OSC.MouthShrugLower", "to": "Standard.MouthShrugLower" },
{ "from": "OSC.MouthClose", "to": "Standard.MouthClose" },
{ "from": "OSC.MouthSmileLeft", "to": "Standard.MouthSmileLeft" },
{ "from": "OSC.MouthSmileRight", "to": "Standard.MouthSmileRight" },
{ "from": "OSC.MouthFrownLeft", "to": "Standard.MouthFrownLeft" },
{ "from": "OSC.MouthFrownRight", "to": "Standard.MouthFrownRight" },
{ "from": "OSC.MouthDimpleLeft", "to": "Standard.MouthDimpleLeft" },
{ "from": "OSC.MouthDimpleRight", "to": "Standard.MouthDimpleRight" },
{ "from": "OSC.MouthUpperUpLeft", "to": "Standard.MouthUpperUpLeft" },
{ "from": "OSC.MouthUpperUpRight", "to": "Standard.MouthUpperUpRight" },
{ "from": "OSC.MouthLowerDownLeft", "to": "Standard.MouthLowerDownLeft" },
{ "from": "OSC.MouthLowerDownRight", "to": "Standard.MouthLowerDownRight" },
{ "from": "OSC.MouthPressLeft", "to": "Standard.MouthPressLeft" },
{ "from": "OSC.MouthPressRight", "to": "Standard.MouthPressRight" },
{ "from": "OSC.MouthStretchLeft", "to": "Standard.MouthStretchLeft" },
{ "from": "OSC.MouthStretchRight", "to": "Standard.MouthStretchRight" },
{ "from": "OSC.TongueOut", "to": "Standard.TongueOut" }
]
}

2
interface/resources/qml/hifi/tablet/ControllerSettings.qml
View file

@ -332,7 +332,7 @@ Item {
anchors.fill: stackView
id: controllerPrefereneces
objectName: "TabletControllerPreferences"
showCategories: ["VR Movement", "Game Controller", "Sixense Controllers", "Perception Neuron", "Leap Motion"]
showCategories: ["VR Movement", "Game Controller", "Sixense Controllers", "Perception Neuron", "Leap Motion", "Open Sound Control (OSC)"]
categoryProperties: {
"VR Movement" : {
"User real-world height (meters)" : { "anchors.right" : "undefined" },

6
plugins/CMakeLists.txt
View file

@ -36,6 +36,12 @@ if (NOT SERVER_ONLY AND NOT ANDROID)
add_subdirectory(${DIR})
set(DIR "hifiLeapMotion")
add_subdirectory(${DIR})
if (WIN32)
set(DIR "hifiOsc")
add_subdirectory(${DIR})
endif()
endif()
# server-side plugins

14
plugins/hifiOsc/CMakeLists.txt Normal file
View file

@ -0,0 +1,14 @@
#
# Created by Anthony Thibault on 2019/8/24
# Copyright 2019 High Fidelity, Inc.
#
# Distributed under the Apache License, Version 2.0.
# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
#
set(TARGET_NAME hifiOsc)
setup_hifi_plugin(Qml)
link_hifi_libraries(shared controllers ui plugins input-plugins display-plugins)
target_liblo()

633
plugins/hifiOsc/src/OscPlugin.cpp Normal file
View file

@ -0,0 +1,633 @@
//
// OscPlugin.cpp
//
// Created by Anthony Thibault on 2019/8/24
// Copyright 2019 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "OscPlugin.h"
#include <controllers/UserInputMapper.h>
#include <QLoggingCategory>
#include <PathUtils.h>
#include <DebugDraw.h>
#include <cassert>
#include <NumericalConstants.h>
#include <StreamUtils.h>
#include <Preferences.h>
#include <SettingHandle.h>
Q_DECLARE_LOGGING_CATEGORY(inputplugins)
Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins")
const char* OscPlugin::NAME = "Open Sound Control (OSC)";
const char* OscPlugin::OSC_ID_STRING = "Open Sound Control (OSC)";
const bool DEFAULT_ENABLED = false;
enum class FaceCap {
BrowsU_C = 0,
BrowsD_L,
BrowsD_R,
BrowsU_L,
BrowsU_R,
EyeUp_L,
EyeUp_R,
EyeDown_L,
EyeDown_R,
EyeIn_L,
EyeIn_R,
EyeOut_L,
EyeOut_R,
EyeBlink_L,
EyeBlink_R,
EyeSquint_L,
EyeSquint_R,
EyeOpen_L,
EyeOpen_R,
Puff,
CheekSquint_L,
CheekSquint_R,
NoseSneer_L,
NoseSneer_R,
JawOpen,
JawFwd,
JawLeft,
JawRight,
LipsFunnel,
LipsPucker,
MouthLeft,
MouthRight,
LipsUpperClose,
LipsLowerClose,
MouthShrugUpper,
MouthShrugLower,
MouthClose,
MouthSmile_L,
MouthSmile_R,
MouthFrown_L,
MouthFrown_R,
MouthDimple_L,
MouthDimple_R,
MouthUpperUp_L,
MouthUpperUp_R,
MouthLowerDown_L,
MouthLowerDown_R,
MouthPress_L,
MouthPress_R,
LipsStretch_L,
LipsStretch_R,
TongueOut,
BlendshapeCount
};
// used to mirror left/right shapes from FaceCap.
// i.e. right and left shapes are swapped.
FaceCap faceMirrorMap[(int)FaceCap::BlendshapeCount] = {
FaceCap::BrowsU_C,
FaceCap::BrowsD_R,
FaceCap::BrowsD_L,
FaceCap::BrowsU_R,
FaceCap::BrowsU_L,
FaceCap::EyeUp_R,
FaceCap::EyeUp_L,
FaceCap::EyeDown_R,
FaceCap::EyeDown_L,
FaceCap::EyeIn_R,
FaceCap::EyeIn_L,
FaceCap::EyeOut_R,
FaceCap::EyeOut_L,
FaceCap::EyeBlink_R,
FaceCap::EyeBlink_L,
FaceCap::EyeSquint_R,
FaceCap::EyeSquint_L,
FaceCap::EyeOpen_R,
FaceCap::EyeOpen_L,
FaceCap::Puff,
FaceCap::CheekSquint_R,
FaceCap::CheekSquint_L,
FaceCap::NoseSneer_R,
FaceCap::NoseSneer_L,
FaceCap::JawOpen,
FaceCap::JawFwd,
FaceCap::JawRight,
FaceCap::JawLeft,
FaceCap::LipsFunnel,
FaceCap::LipsPucker,
FaceCap::MouthRight,
FaceCap::MouthLeft,
FaceCap::LipsUpperClose,
FaceCap::LipsLowerClose,
FaceCap::MouthShrugUpper,
FaceCap::MouthShrugLower,
FaceCap::MouthClose,
FaceCap::MouthSmile_R,
FaceCap::MouthSmile_L,
FaceCap::MouthFrown_R,
FaceCap::MouthFrown_L,
FaceCap::MouthDimple_R,
FaceCap::MouthDimple_L,
FaceCap::MouthUpperUp_R,
FaceCap::MouthUpperUp_L,
FaceCap::MouthLowerDown_R,
FaceCap::MouthLowerDown_L,
FaceCap::MouthPress_R,
FaceCap::MouthPress_L,
FaceCap::LipsStretch_R,
FaceCap::LipsStretch_L,
FaceCap::TongueOut
};
static const char* STRINGS[FaceCap::BlendshapeCount] = {
"BrowsU_C",
"BrowsD_L",
"BrowsD_R",
"BrowsU_L",
"BrowsU_R",
"EyeUp_L",
"EyeUp_R",
"EyeDown_L",
"EyeDown_R",
"EyeIn_L",
"EyeIn_R",
"EyeOut_L",
"EyeOut_R",
"EyeBlink_L",
"EyeBlink_R",
"EyeSquint_L",
"EyeSquint_R",
"EyeOpen_L",
"EyeOpen_R",
"Puff",
"CheekSquint_L",
"CheekSquint_R",
"NoseSneer_L",
"NoseSneer_R",
"JawOpen",
"JawFwd",
"JawLeft",
"JawRight",
"LipsFunnel",
"LipsPucker",
"MouthLeft",
"MouthRight",
"LipsUpperClose",
"LipsLowerClose",
"MouthShrugUpper",
"MouthShrugLower",
"MouthClose",
"MouthSmile_L",
"MouthSmile_R",
"MouthFrown_L",
"MouthFrown_R",
"MouthDimple_L",
"MouthDimple_R",
"MouthUpperUp_L",
"MouthUpperUp_R",
"MouthLowerDown_L",
"MouthLowerDown_R",
"MouthPress_L",
"MouthPress_R",
"LipsStretch_L",
"LipsStretch_R",
"TongueOut"
};
static enum controller::StandardAxisChannel CHANNELS[FaceCap::BlendshapeCount] = {
controller::BROWSU_C,
controller::BROWSD_L,
controller::BROWSD_R,
controller::BROWSU_L,
controller::BROWSU_R,
controller::EYEUP_L,
controller::EYEUP_R,
controller::EYEDOWN_L,
controller::EYEDOWN_R,
controller::EYEIN_L,
controller::EYEIN_R,
controller::EYEOUT_L,
controller::EYEOUT_R,
controller::EYEBLINK_L,
controller::EYEBLINK_R,
controller::EYESQUINT_L,
controller::EYESQUINT_R,
controller::EYEOPEN_L,
controller::EYEOPEN_R,
controller::PUFF,
controller::CHEEKSQUINT_L,
controller::CHEEKSQUINT_R,
controller::NOSESNEER_L,
controller::NOSESNEER_R,
controller::JAWOPEN,
controller::JAWFWD,
controller::JAWLEFT,
controller::JAWRIGHT,
controller::LIPSFUNNEL,
controller::LIPSPUCKER,
controller::MOUTHLEFT,
controller::MOUTHRIGHT,
controller::LIPSUPPERCLOSE,
controller::LIPSLOWERCLOSE,
controller::MOUTHSHRUGUPPER,
controller::MOUTHSHRUGLOWER,
controller::MOUTHCLOSE,
controller::MOUTHSMILE_L,
controller::MOUTHSMILE_R,
controller::MOUTHFROWN_L,
controller::MOUTHFROWN_R,
controller::MOUTHDIMPLE_L,
controller::MOUTHDIMPLE_R,
controller::MOUTHUPPERUP_L,
controller::MOUTHUPPERUP_R,
controller::MOUTHLOWERDOWN_L,
controller::MOUTHLOWERDOWN_R,
controller::MOUTHPRESS_L,
controller::MOUTHPRESS_R,
controller::LIPSSTRETCH_L,
controller::LIPSSTRETCH_R,
controller::TONGUEOUT
};
void OscPlugin::init() {
_inputDevice = std::make_shared<InputDevice>();
_inputDevice->setContainer(this);
{
std::lock_guard<std::mutex> guard(_dataMutex);
_blendshapeValues.assign((int)FaceCap::BlendshapeCount, 0.0f);
_blendshapeValidFlags.assign((int)FaceCap::BlendshapeCount, false);
_headRot = glm::quat();
_headRotValid = false;
_headTransTarget = extractTranslation(_lastInputCalibrationData.defaultHeadMat);
_headTransSmoothed = extractTranslation(_lastInputCalibrationData.defaultHeadMat);
_headTransValid = false;
_eyeLeftRot = glm::quat();
_eyeLeftRotValid = false;
_eyeRightRot = glm::quat();
_eyeRightRotValid = false;
}
loadSettings();
auto preferences = DependencyManager::get<Preferences>();
static const QString OSC_PLUGIN { OscPlugin::NAME };
{
auto getter = [this]()->bool { return _enabled; };
auto setter = [this](bool value) {
_enabled = value;
saveSettings();
if (!_enabled) {
auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
userInputMapper->withLock([&, this]() {
_inputDevice->clearState();
restartServer();
});
}
};
auto preference = new CheckPreference(OSC_PLUGIN, "Enabled", getter, setter);
preferences->addPreference(preference);
}
{
auto debugGetter = [this]()->bool { return _debug; };
auto debugSetter = [this](bool value) {
_debug = value;
saveSettings();
};
auto preference = new CheckPreference(OSC_PLUGIN, "Extra Debugging", debugGetter, debugSetter);
preferences->addPreference(preference);
}
{
static const int MIN_PORT_NUMBER { 0 };
static const int MAX_PORT_NUMBER { 65535 };
auto getter = [this]()->float { return (float)_serverPort; };
auto setter = [this](float value) {
_serverPort = (int)value;
saveSettings();
restartServer();
};
auto preference = new SpinnerPreference(OSC_PLUGIN, "Server Port", getter, setter);
preference->setMin(MIN_PORT_NUMBER);
preference->setMax(MAX_PORT_NUMBER);
preference->setStep(1);
preferences->addPreference(preference);
}
}
bool OscPlugin::isSupported() const {
// networking/UDP is pretty much always available...
return true;
}
static void errorHandlerFunc(int num, const char* msg, const char* path) {
qDebug(inputplugins) << "OscPlugin: server error" << num << "in path" << path << ":" << msg;
}
static int genericHandlerFunc(const char* path, const char* types, lo_arg** argv,
int argc, void* data, void* user_data) {
OscPlugin* container = reinterpret_cast<OscPlugin*>(user_data);
assert(container);
QString key(path);
std::lock_guard<std::mutex> guard(container->_dataMutex);
// Special case: decode blendshapes from face-cap iPhone app.
// http://www.bannaflak.com/face-cap/livemode.html
if (path[0] == '/' && path[1] == 'W' && argc == 2 && types[0] == 'i' && types[1] == 'f') {
int index = argv[0]->i;
if (index >= 0 && index < (int)FaceCap::BlendshapeCount) {
int mirroredIndex = (int)faceMirrorMap[index];
container->_blendshapeValues[mirroredIndex] = argv[1]->f;
container->_blendshapeValidFlags[mirroredIndex] = true;
}
}
// map /HT to head translation
if (path[0] == '/' && path[1] == 'H' && path[2] == 'T' &&
types[0] == 'f' && types[1] == 'f' && types[2] == 'f') {
glm::vec3 trans(-argv[0]->f, -argv[1]->f, argv[2]->f); // in cm
// convert trans into a delta (in meters) from the sweet spot of the iphone camera.
const float CM_TO_METERS = 0.01f;
const glm::vec3 FACE_CAP_HEAD_SWEET_SPOT(0.0f, 0.0f, -45.0f);
glm::vec3 delta = (trans - FACE_CAP_HEAD_SWEET_SPOT) * CM_TO_METERS;
container->_headTransTarget = extractTranslation(container->_lastInputCalibrationData.defaultHeadMat) + delta;
container->_headTransValid = true;
}
// map /HR to head rotation
if (path[0] == '/' && path[1] == 'H' && path[2] == 'R' && path[3] == 0 &&
types[0] == 'f' && types[1] == 'f' && types[2] == 'f') {
glm::vec3 euler(-argv[0]->f, -argv[1]->f, argv[2]->f);
container->_headRot = glm::quat(glm::radians(euler)) * Quaternions::Y_180;
container->_headRotValid = true;
}
// map /ELR to left eye rot
if (path[0] == '/' && path[1] == 'E' && path[2] == 'L' && path[3] == 'R' &&
types[0] == 'f' && types[1] == 'f' && types[2] == 'f') {
glm::vec3 euler(-argv[0]->f, -argv[1]->f, argv[2]->f);
container->_eyeLeftRot = glm::quat(glm::radians(euler)) * Quaternions::Y_180;
container->_eyeLeftRotValid = true;
}
// map /ERR to right eye rot
if (path[0] == '/' && path[1] == 'E' && path[2] == 'R' && path[3] == 'R' &&
types[0] == 'f' && types[1] == 'f' && types[2] == 'f') {
glm::vec3 euler(-argv[0]->f, -argv[1]->f, argv[2]->f);
container->_eyeRightRot = glm::quat(glm::radians(euler)) * Quaternions::Y_180;
container->_eyeRightRotValid = true;
}
// AJT: TODO map /STRINGS[i] to _blendshapeValues[i]
if (container->_debug) {
for (int i = 0; i < argc; i++) {
switch (types[i]) {
case 'i':
// int32
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->i;
break;
case 'f':
// float32
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->f32;
break;
case 's':
// OSC-string
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <string>";
break;
case 'b':
// OSC-blob
break;
case 'h':
// 64 bit big-endian two's complement integer
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->h;
break;
case 't':
// OSC-timetag
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <OSC-timetag>";
break;
case 'd':
// 64 bit ("double") IEEE 754 floating point number
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->d;
break;
case 'S':
// Alternate type represented as an OSC-string (for example, for systems that differentiate "symbols" from "strings")
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <OSC-symbol>";
break;
case 'c':
// an ascii character, sent as 32 bits
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] =" << argv[i]->c;
break;
case 'r':
// 32 bit RGBA color
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <color>";
break;
case 'm':
// 4 byte MIDI message. Bytes from MSB to LSB are: port id, status byte, data1, data2
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <midi>";
break;
case 'T':
// true
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <true>";
break;
case 'F':
// false
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <false>";
break;
case 'N':
// nil
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <nil>";
break;
case 'I':
// inf
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <inf>";
break;
case '[':
// Indicates the beginning of an array. The tags following are for data in the Array until a close brace tag is reached.
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <begin-array>";
break;
case ']':
// Indicates the end of an array.
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <end-array>";
break;
default:
qDebug(inputplugins) << "OscPlugin: " << path << "[" << i << "] = <unknown-type>" << types[i];
break;
}
}
}
return 1;
}
bool OscPlugin::activate() {
InputPlugin::activate();
loadSettings();
if (_enabled) {
qDebug(inputplugins) << "OscPlugin: activated";
// register with userInputMapper
auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
userInputMapper->registerDevice(_inputDevice);
return startServer();
}
return false;
}
void OscPlugin::deactivate() {
qDebug(inputplugins) << "OscPlugin: deactivated, _oscServerThread =" << _oscServerThread;
if (_oscServerThread) {
stopServer();
}
}
void OscPlugin::pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
if (!_enabled) {
return;
}
_lastInputCalibrationData = inputCalibrationData;
auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
userInputMapper->withLock([&, this]() {
_inputDevice->update(deltaTime, inputCalibrationData);
});
}
void OscPlugin::saveSettings() const {
Settings settings;
QString idString = getID();
settings.beginGroup(idString);
{
settings.setValue(QString("enabled"), _enabled);
settings.setValue(QString("extraDebug"), _debug);
settings.setValue(QString("serverPort"), _serverPort);
}
settings.endGroup();
}
void OscPlugin::loadSettings() {
Settings settings;
QString idString = getID();
settings.beginGroup(idString);
{
_enabled = settings.value("enabled", QVariant(DEFAULT_ENABLED)).toBool();
_debug = settings.value("extraDebug", QVariant(DEFAULT_ENABLED)).toBool();
_serverPort = settings.value("serverPort", QVariant(DEFAULT_OSC_SERVER_PORT)).toInt();
}
settings.endGroup();
}
bool OscPlugin::startServer() {
if (_oscServerThread) {
qWarning(inputplugins) << "OscPlugin: (startServer) server is already running, _oscServerThread =" << _oscServerThread;
return true;
}
// start a new server on specified port
const size_t BUFFER_SIZE = 64;
char serverPortString[BUFFER_SIZE];
snprintf(serverPortString, BUFFER_SIZE, "%d", _serverPort);
_oscServerThread = lo_server_thread_new(serverPortString, errorHandlerFunc);
qDebug(inputplugins) << "OscPlugin: server started on port" << serverPortString << ", _oscServerThread =" << _oscServerThread;
// add method that will match any path and args
// NOTE: callback function will be called on the OSC thread, not the appliation thread.
lo_server_thread_add_method(_oscServerThread, NULL, NULL, genericHandlerFunc, (void*)this);
lo_server_thread_start(_oscServerThread);
return true;
}
void OscPlugin::stopServer() {
if (!_oscServerThread) {
qWarning(inputplugins) << "OscPlugin: (stopServer) server is already shutdown.";
}
// stop and free server
lo_server_thread_stop(_oscServerThread);
lo_server_thread_free(_oscServerThread);
_oscServerThread = nullptr;
}
void OscPlugin::restartServer() {
if (_oscServerThread) {
stopServer();
}
startServer();
}
//
// InputDevice
//
controller::Input::NamedVector OscPlugin::InputDevice::getAvailableInputs() const {
static controller::Input::NamedVector availableInputs;
if (availableInputs.size() == 0) {
for (int i = 0; i < (int)FaceCap::BlendshapeCount; i++) {
availableInputs.push_back(makePair(CHANNELS[i], STRINGS[i]));
}
}
availableInputs.push_back(makePair(controller::HEAD, "Head"));
availableInputs.push_back(makePair(controller::LEFT_EYE, "LeftEye"));
availableInputs.push_back(makePair(controller::RIGHT_EYE, "RightEye"));
return availableInputs;
}
QString OscPlugin::InputDevice::getDefaultMappingConfig() const {
static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/osc.json";
return MAPPING_JSON;
}
void OscPlugin::InputDevice::update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
std::lock_guard<std::mutex> guard(_container->_dataMutex);
for (int i = 0; i < (int)FaceCap::BlendshapeCount; i++) {
if (_container->_blendshapeValidFlags[i]) {
_axisStateMap[CHANNELS[i]] = controller::AxisValue(_container->_blendshapeValues[i], 0, true);
}
}
if (_container->_headRotValid && _container->_headTransValid) {
const float SMOOTH_TIMESCALE = 2.0f;
float tau = deltaTime / SMOOTH_TIMESCALE;
_container->_headTransSmoothed = lerp(_container->_headTransSmoothed, _container->_headTransTarget, tau);
glm::vec3 delta = _container->_headTransSmoothed - _container->_headTransTarget;
glm::vec3 trans = extractTranslation(inputCalibrationData.defaultHeadMat) + delta;
_poseStateMap[controller::HEAD] = controller::Pose(trans, _container->_headRot);
}
if (_container->_eyeLeftRotValid) {
_poseStateMap[controller::LEFT_EYE] = controller::Pose(vec3(0.0f), _container->_eyeLeftRot);
}
if (_container->_eyeRightRotValid) {
_poseStateMap[controller::RIGHT_EYE] = controller::Pose(vec3(0.0f), _container->_eyeRightRot);
}
}
void OscPlugin::InputDevice::clearState() {
std::lock_guard<std::mutex> guard(_container->_dataMutex);
for (int i = 0; i < (int)FaceCap::BlendshapeCount; i++) {
_axisStateMap[CHANNELS[i]] = controller::AxisValue(0.0f, 0, false);
}
_poseStateMap[controller::HEAD] = controller::Pose();
_poseStateMap[controller::LEFT_EYE] = controller::Pose();
_poseStateMap[controller::RIGHT_EYE] = controller::Pose();
}

95
plugins/hifiOsc/src/OscPlugin.h Normal file
View file

@ -0,0 +1,95 @@
//
// OscPlugin.h
//
// Created by Anthony Thibault on 2019/8/24
// Copyright 2019 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_OscPlugin_h
#define hifi_OscPlugin_h
#include <controllers/InputDevice.h>
#include <controllers/StandardControls.h>
#include <plugins/InputPlugin.h>
#include "lo/lo.h"
// OSC (Open Sound Control) input plugin.
class OscPlugin : public InputPlugin {
Q_OBJECT
public:
// Plugin functions
virtual void init() override;
virtual bool isSupported() const override;
virtual const QString getName() const override { return NAME; }
const QString getID() const override { return OSC_ID_STRING; }
virtual bool activate() override;
virtual void deactivate() override;
virtual void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); }
virtual void pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override;
virtual void saveSettings() const override;
virtual void loadSettings() override;
bool startServer();
void stopServer();
void restartServer();
protected:
class InputDevice : public controller::InputDevice {
public:
friend class OscPlugin;
InputDevice() : controller::InputDevice("OSC") {}
// Device functions
virtual controller::Input::NamedVector getAvailableInputs() const override;
virtual QString getDefaultMappingConfig() const override;
virtual void update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override;
virtual void focusOutEvent() override {};
void clearState();
void setContainer(OscPlugin* container) { _container = container; }
OscPlugin* _container { nullptr };
};
std::shared_ptr<InputDevice> _inputDevice;
static const char* NAME;
static const char* OSC_ID_STRING;
bool _enabled { false };
mutable bool _initialized { false };
lo_server_thread _oscServerThread { nullptr };
public:
bool _debug { false };
enum Constants { DEFAULT_OSC_SERVER_PORT = 7700 };
int _serverPort { DEFAULT_OSC_SERVER_PORT };
controller::InputCalibrationData _lastInputCalibrationData;
std::vector<float> _blendshapeValues;
std::vector<bool> _blendshapeValidFlags;
glm::quat _headRot;
bool _headRotValid { false };
glm::vec3 _headTransTarget;
glm::vec3 _headTransSmoothed;
bool _headTransValid { false };
glm::quat _eyeLeftRot;
bool _eyeLeftRotValid { false };
glm::quat _eyeRightRot;
bool _eyeRightRotValid { false };
std::mutex _dataMutex;
};
#endif // hifi_OscPlugin_h

49
plugins/hifiOsc/src/OscProvider.cpp Normal file
View file

@ -0,0 +1,49 @@
//
// Created by Anthony Thibault on 2019/8/25
// Copyright 2019 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <mutex>
#include <QtCore/QObject>
#include <QtCore/QtPlugin>
#include <QtCore/QStringList>
#include <plugins/RuntimePlugin.h>
#include <plugins/InputPlugin.h>
#include "OscPlugin.h"
class OscProvider : public QObject, public InputProvider
{
Q_OBJECT
Q_PLUGIN_METADATA(IID InputProvider_iid FILE "plugin.json")
Q_INTERFACES(InputProvider)
public:
OscProvider(QObject* parent = nullptr) : QObject(parent) {}
virtual ~OscProvider() {}
virtual InputPluginList getInputPlugins() override {
static std::once_flag once;
std::call_once(once, [&] {
InputPluginPointer plugin(new OscPlugin());
if (plugin->isSupported()) {
_inputPlugins.push_back(plugin);
}
});
return _inputPlugins;
}
virtual void destroyInputPlugins() override {
_inputPlugins.clear();
}
private:
InputPluginList _inputPlugins;
};
#include "OscProvider.moc"

4
plugins/hifiOsc/src/plugin.json Normal file
View file

@ -0,0 +1,4 @@
{
"name":"Osc",
"version": 1
}