diff --git a/cmake/externals/neuron/CMakeLists.txt b/cmake/externals/neuron/CMakeLists.txt new file mode 100644 index 0000000000..6936725571 --- /dev/null +++ b/cmake/externals/neuron/CMakeLists.txt @@ -0,0 +1,56 @@ +include(ExternalProject) + +set(EXTERNAL_NAME neuron) + +string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) + +set(NEURON_URL "https://s3.amazonaws.com/hifi-public/dependencies/neuron_datareader_b.12.zip") +set(NEURON_URL_MD5 "0ab54ca04c9cc8094e0fa046c226e574") + +ExternalProject_Add(${EXTERNAL_NAME} + URL ${NEURON_URL} + URL_MD5 ${NEURON_URL_MD5} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + LOG_DOWNLOAD 1) + +ExternalProject_Get_Property(${EXTERNAL_NAME} SOURCE_DIR) + +set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals") + +# set include dir +if(WIN32) + set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS "${SOURCE_DIR}/NeuronDataReader_Windows/include" CACHE TYPE INTERNAL) +elseif(APPLE) + set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS "${SOURCE_DIR}/NeuronDataReader_Mac/include" CACHE TYPE INTERNAL) +else() + # Unsupported +endif() + +if(WIN32) + + if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + set(ARCH_DIR "x64") + else() + set(ARCH_DIR "x86") + endif() + + set(${EXTERNAL_NAME_UPPER}_LIB_PATH "${SOURCE_DIR}/NeuronDataReader_Windows/lib/${ARCH_DIR}") + set(${EXTERNAL_NAME_UPPER}_LIBRARY_RELEASE "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.lib" CACHE TYPE INTERNAL) + set(${EXTERNAL_NAME_UPPER}_LIBRARIES "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.lib" CACHE TYPE INTERNAL) + + add_paths_to_fixup_libs("${${EXTERNAL_NAME_UPPER}_LIB_PATH}") + +elseif(APPLE) + + set(${EXTERNAL_NAME_UPPER}_LIB_PATH "${SOURCE_DIR}/NeuronDataReader_Mac/dylib") + set(${EXTERNAL_NAME_UPPER}_LIBRARY_RELEASE "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.dylib" CACHE TYPE INTERNAL) + set(${EXTERNAL_NAME_UPPER}_LIBRARIES "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.dylib" CACHE TYPE INTERNAL) + + add_paths_to_fixup_libs("${${EXTERNAL_NAME_UPPER}_LIB_PATH}") + +else() + # UNSUPPORTED +endif() + diff --git a/cmake/macros/TargetNeuron.cmake b/cmake/macros/TargetNeuron.cmake new file mode 100644 index 0000000000..11a92e68ad --- /dev/null +++ b/cmake/macros/TargetNeuron.cmake @@ -0,0 +1,17 @@ +# +# Copyright 2015 High Fidelity, Inc. +# Created by Anthony J. Thibault on 2015/12/21 +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# +macro(TARGET_NEURON) + # Neuron data reader is only available on these platforms + if (WIN32 OR APPLE) + add_dependency_external_projects(neuron) + find_package(Neuron REQUIRED) + target_include_directories(${TARGET_NAME} PRIVATE ${NEURON_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${NEURON_LIBRARIES}) + add_definitions(-DHAVE_NEURON) + endif(WIN32 OR APPLE) +endmacro() diff --git a/cmake/modules/FindNeuron.cmake b/cmake/modules/FindNeuron.cmake new file mode 100644 index 0000000000..6a93b3a016 --- /dev/null +++ b/cmake/modules/FindNeuron.cmake @@ -0,0 +1,28 @@ +# +# FindNeuron.cmake +# +# Try to find the Perception Neuron SDK +# +# You must provide a NEURON_ROOT_DIR which contains lib and include directories +# +# Once done this will define +# +# NEURON_FOUND - system found Neuron SDK +# NEURON_INCLUDE_DIRS - the Neuron SDK include directory +# NEURON_LIBRARIES - Link this to use Neuron +# +# Created on 12/21/2015 by Anthony J. Thibault +# Copyright 2015 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# + +include(SelectLibraryConfigurations) +select_library_configurations(NEURON) + +set(NEURON_REQUIREMENTS NEURON_INCLUDE_DIRS NEURON_LIBRARIES) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Neuron DEFAULT_MSG NEURON_INCLUDE_DIRS NEURON_LIBRARIES) +mark_as_advanced(NEURON_LIBRARIES NEURON_INCLUDE_DIRS NEURON_SEARCH_DIRS) + diff --git a/examples/controllers/neuron/neuronAvatar.js b/examples/controllers/neuron/neuronAvatar.js new file mode 100644 index 0000000000..7cebc13feb --- /dev/null +++ b/examples/controllers/neuron/neuronAvatar.js @@ -0,0 +1,239 @@ +var JOINT_PARENT_MAP = { + Hips: "", + RightUpLeg: "Hips", + RightLeg: "RightUpLeg", + RightFoot: "RightLeg", + LeftUpLeg: "Hips", + LeftLeg: "LeftUpLeg", + LeftFoot: "LeftLeg", + Spine: "Hips", + Spine1: "Spine", + Spine2: "Spine1", + Spine3: "Spine2", + Neck: "Spine3", + Head: "Neck", + RightShoulder: "Spine3", + RightArm: "RightShoulder", + RightForeArm: "RightArm", + RightHand: "RightForeArm", + RightHandThumb1: "RightHand", + RightHandThumb2: "RightHandThumb1", + RightHandThumb3: "RightHandThumb2", + RightHandThumb4: "RightHandThumb3", + RightHandIndex1: "RightHand", + RightHandIndex2: "RightHandIndex1", + RightHandIndex3: "RightHandIndex2", + RightHandIndex4: "RightHandIndex3", + RightHandMiddle1: "RightHand", + RightHandMiddle2: "RightHandMiddle1", + RightHandMiddle3: "RightHandMiddle2", + RightHandMiddle4: "RightHandMiddle3", + RightHandRing1: "RightHand", + RightHandRing2: "RightHandRing1", + RightHandRing3: "RightHandRing2", + RightHandRing4: "RightHandRing3", + RightHandPinky1: "RightHand", + RightHandPinky2: "RightHandPinky1", + RightHandPinky3: "RightHandPinky2", + RightHandPinky4: "RightHandPinky3", + LeftShoulder: "Spine3", + LeftArm: "LeftShoulder", + LeftForeArm: "LeftArm", + LeftHand: "LeftForeArm", + LeftHandThumb1: "LeftHand", + LeftHandThumb2: "LeftHandThumb1", + LeftHandThumb3: "LeftHandThumb2", + LeftHandThumb4: "LeftHandThumb3", + LeftHandIndex1: "LeftHand", + LeftHandIndex2: "LeftHandIndex1", + LeftHandIndex3: "LeftHandIndex2", + LeftHandIndex4: "LeftHandIndex3", + LeftHandMiddle1: "LeftHand", + LeftHandMiddle2: "LeftHandMiddle1", + LeftHandMiddle3: "LeftHandMiddle2", + LeftHandMiddle4: "LeftHandMiddle3", + LeftHandRing1: "LeftHand", + LeftHandRing2: "LeftHandRing1", + LeftHandRing3: "LeftHandRing2", + LeftHandRing4: "LeftHandRing3", + LeftHandPinky1: "LeftHand", + LeftHandPinky2: "LeftHandPinky1", + LeftHandPinky3: "LeftHandPinky2", + LeftHandPinky: "LeftHandPinky3", +}; + +var USE_TRANSLATIONS = false; + +// ctor +function Xform(rot, pos) { + this.rot = rot; + this.pos = pos; +}; +Xform.mul = function (lhs, rhs) { + var rot = Quat.multiply(lhs.rot, rhs.rot); + var pos = Vec3.sum(lhs.pos, Vec3.multiplyQbyV(lhs.rot, rhs.pos)); + return new Xform(rot, pos); +}; +Xform.prototype.inv = function () { + var invRot = Quat.inverse(this.rot); + var invPos = Vec3.multiply(-1, this.pos); + return new Xform(invRot, Vec3.multiplyQbyV(invRot, invPos)); +}; +Xform.prototype.toString = function () { + var rot = this.rot; + var pos = this.pos; + return "Xform rot = (" + rot.x + ", " + rot.y + ", " + rot.z + ", " + rot.w + "), pos = (" + pos.x + ", " + pos.y + ", " + pos.z + ")"; +}; + +function dumpHardwareMapping() { + Object.keys(Controller.Hardware).forEach(function (deviceName) { + Object.keys(Controller.Hardware[deviceName]).forEach(function (input) { + print("Controller.Hardware." + deviceName + "." + input + ":" + Controller.Hardware[deviceName][input]); + }); + }); +} + +// ctor +function NeuronAvatar() { + var self = this; + Script.scriptEnding.connect(function () { + self.shutdown(); + }); + Controller.hardwareChanged.connect(function () { + self.hardwareChanged(); + }); + + if (Controller.Hardware.Neuron) { + this.activate(); + } else { + this.deactivate(); + } +} + +NeuronAvatar.prototype.shutdown = function () { + this.deactivate(); +}; + +NeuronAvatar.prototype.hardwareChanged = function () { + if (Controller.Hardware.Neuron) { + this.activate(); + } else { + this.deactivate(); + } +}; + +NeuronAvatar.prototype.activate = function () { + if (!this._active) { + Script.update.connect(updateCallback); + } + this._active = true; + + // build absDefaultPoseMap + this._defaultAbsRotMap = {}; + this._defaultAbsPosMap = {}; + this._defaultAbsRotMap[""] = {x: 0, y: 0, z: 0, w: 1}; + this._defaultAbsPosMap[""] = {x: 0, y: 0, z: 0}; + var keys = Object.keys(JOINT_PARENT_MAP); + var i, l = keys.length; + for (i = 0; i < l; i++) { + var jointName = keys[i]; + var j = MyAvatar.getJointIndex(jointName); + var parentJointName = JOINT_PARENT_MAP[jointName]; + this._defaultAbsRotMap[jointName] = Quat.multiply(this._defaultAbsRotMap[parentJointName], MyAvatar.getDefaultJointRotation(j)); + this._defaultAbsPosMap[jointName] = Vec3.sum(this._defaultAbsPosMap[parentJointName], + Quat.multiply(this._defaultAbsRotMap[parentJointName], MyAvatar.getDefaultJointTranslation(j))); + } +}; + +NeuronAvatar.prototype.deactivate = function () { + if (this._active) { + var self = this; + Script.update.disconnect(updateCallback); + } + this._active = false; + MyAvatar.clearJointsData(); +}; + +NeuronAvatar.prototype.update = function (deltaTime) { + + var hmdActive = HMD.active; + var keys = Object.keys(JOINT_PARENT_MAP); + var i, l = keys.length; + var absDefaultRot = {}; + var jointName, channel, pose, parentJointName, j, parentDefaultAbsRot; + var localRotations = {}; + var localTranslations = {}; + for (i = 0; i < l; i++) { + var jointName = keys[i]; + var channel = Controller.Hardware.Neuron[jointName]; + if (channel) { + pose = Controller.getPoseValue(channel); + parentJointName = JOINT_PARENT_MAP[jointName]; + j = MyAvatar.getJointIndex(jointName); + defaultAbsRot = this._defaultAbsRotMap[jointName]; + parentDefaultAbsRot = this._defaultAbsRotMap[parentJointName]; + + // Rotations from the neuron controller are in world orientation but are delta's from the default pose. + // So first we build the absolute rotation of the default pose (local into world). + // Then apply the rotation from the controller, in world space. + // Then we transform back into joint local by multiplying by the inverse of the parents absolute rotation. + var localRotation = Quat.multiply(Quat.inverse(parentDefaultAbsRot), Quat.multiply(pose.rotation, defaultAbsRot)); + if (!hmdActive || jointName !== "Hips") { + MyAvatar.setJointRotation(j, localRotation); + } + localRotations[jointName] = localRotation; + + // translation proportions might be different from the neuron avatar and the user avatar skeleton. + // so this is disabled by default + if (USE_TRANSLATIONS) { + var localTranslation = Vec3.multiplyQbyV(Quat.inverse(parentDefaultAbsRot), pose.translation); + MyAvatar.setJointTranslation(j, localTranslation); + localTranslations[jointName] = localTranslation; + } else { + localTranslations[jointName] = MyAvatar.getDefaultJointTranslation(j); + } + } + } + + // it attempts to adjust the hips so that the avatar's head is at the same location & oreintation as the HMD. + // however it's fighting with the internal c++ code that also attempts to adjust the hips. + if (hmdActive) { + var UNIT_SCALE = 1 / 100; + var hmdXform = new Xform(HMD.orientation, Vec3.multiply(1 / UNIT_SCALE, HMD.position)); // convert to cm + var y180Xform = new Xform({x: 0, y: 1, z: 0, w: 0}, {x: 0, y: 0, z: 0}); + var avatarXform = new Xform(MyAvatar.orientation, Vec3.multiply(1 / UNIT_SCALE, MyAvatar.position)); // convert to cm + var hipsJointIndex = MyAvatar.getJointIndex("Hips"); + var modelOffsetInvXform = new Xform({x: 0, y: 0, z: 0, w: 1}, MyAvatar.getDefaultJointTranslation(hipsJointIndex)); + var defaultHipsXform = new Xform(MyAvatar.getDefaultJointRotation(hipsJointIndex), MyAvatar.getDefaultJointTranslation(hipsJointIndex)); + + var headXform = new Xform(localRotations["Head"], localTranslations["Head"]); + + // transform eyes down the heirarchy chain into avatar space. + var hierarchy = ["Neck", "Spine3", "Spine2", "Spine1", "Spine"]; + var i, l = hierarchy.length; + for (i = 0; i < l; i++) { + var xform = new Xform(localRotations[hierarchy[i]], localTranslations[hierarchy[i]]); + headXform = Xform.mul(xform, headXform); + } + headXform = Xform.mul(defaultHipsXform, headXform); + + var preXform = Xform.mul(headXform, y180Xform); + var postXform = Xform.mul(avatarXform, Xform.mul(y180Xform, modelOffsetInvXform.inv())); + + // solve for the offset that will put the eyes at the hmd position & orientation. + var hipsOffsetXform = Xform.mul(postXform.inv(), Xform.mul(hmdXform, preXform.inv())); + + // now combine it with the default hips transform + var hipsXform = Xform.mul(hipsOffsetXform, defaultHipsXform); + + MyAvatar.setJointRotation("Hips", hipsXform.rot); + MyAvatar.setJointTranslation("Hips", hipsXform.pos); + } +}; + +var neuronAvatar = new NeuronAvatar(); + +function updateCallback(deltaTime) { + neuronAvatar.update(deltaTime); +} + diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 1d9557a835..5d96b95624 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -109,7 +109,9 @@ add_dependency_external_projects(sdl2) if (WIN32) add_dependency_external_projects(OpenVR) endif() - +if(WIN32 OR APPLE) + add_dependency_external_projects(neuron) +endif() # disable /OPT:REF and /OPT:ICF for the Debug builds # This will prevent the following linker warnings diff --git a/interface/resources/controllers/neuron.json b/interface/resources/controllers/neuron.json new file mode 100644 index 0000000000..2d61f80c35 --- /dev/null +++ b/interface/resources/controllers/neuron.json @@ -0,0 +1,7 @@ +{ + "name": "Neuron to Standard", + "channels": [ + { "from": "Hydra.LeftHand", "to": "Standard.LeftHand" }, + { "from": "Hydra.RightHand", "to": "Standard.RightHand" } + ] +} diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index 68f382d2d9..4dd091f1d6 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -289,8 +289,10 @@ void Rig::clearJointState(int index) { void Rig::clearJointStates() { _internalPoseSet._overrideFlags.clear(); - _internalPoseSet._overrideFlags.resize(_animSkeleton->getNumJoints()); - _internalPoseSet._overridePoses = _animSkeleton->getRelativeDefaultPoses(); + if (_animSkeleton) { + _internalPoseSet._overrideFlags.resize(_animSkeleton->getNumJoints()); + _internalPoseSet._overridePoses = _animSkeleton->getRelativeDefaultPoses(); + } } void Rig::clearJointAnimationPriority(int index) { diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index bbd33c5cb3..2b0613321e 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -88,15 +88,73 @@ namespace controller { // No correlation to SDL enum StandardPoseChannel { - LEFT_HAND = 0, - RIGHT_HAND, + HIPS = 0, + RIGHT_UP_LEG, + RIGHT_LEG, + RIGHT_FOOT, + LEFT_UP_LEG, + LEFT_LEG, + LEFT_FOOT, + SPINE, + SPINE1, + SPINE2, + SPINE3, + NECK, HEAD, + RIGHT_SHOULDER, + RIGHT_ARM, + RIGHT_FORE_ARM, + RIGHT_HAND, + RIGHT_HAND_THUMB1, + RIGHT_HAND_THUMB2, + RIGHT_HAND_THUMB3, + RIGHT_HAND_THUMB4, + RIGHT_HAND_INDEX1, + RIGHT_HAND_INDEX2, + RIGHT_HAND_INDEX3, + RIGHT_HAND_INDEX4, + RIGHT_HAND_MIDDLE1, + RIGHT_HAND_MIDDLE2, + RIGHT_HAND_MIDDLE3, + RIGHT_HAND_MIDDLE4, + RIGHT_HAND_RING1, + RIGHT_HAND_RING2, + RIGHT_HAND_RING3, + RIGHT_HAND_RING4, + RIGHT_HAND_PINKY1, + RIGHT_HAND_PINKY2, + RIGHT_HAND_PINKY3, + RIGHT_HAND_PINKY4, + LEFT_SHOULDER, + LEFT_ARM, + LEFT_FORE_ARM, + LEFT_HAND, + LEFT_HAND_THUMB1, + LEFT_HAND_THUMB2, + LEFT_HAND_THUMB3, + LEFT_HAND_THUMB4, + LEFT_HAND_INDEX1, + LEFT_HAND_INDEX2, + LEFT_HAND_INDEX3, + LEFT_HAND_INDEX4, + LEFT_HAND_MIDDLE1, + LEFT_HAND_MIDDLE2, + LEFT_HAND_MIDDLE3, + LEFT_HAND_MIDDLE4, + LEFT_HAND_RING1, + LEFT_HAND_RING2, + LEFT_HAND_RING3, + LEFT_HAND_RING4, + LEFT_HAND_PINKY1, + LEFT_HAND_PINKY2, + LEFT_HAND_PINKY3, + LEFT_HAND_PINKY4, NUM_STANDARD_POSES }; enum StandardCounts { TRIGGERS = 2, ANALOG_STICKS = 2, - POSES = 2, // FIXME 3? if we want to expose the head? + POSES = NUM_STANDARD_POSES }; } diff --git a/plugins/hifiNeuron/CMakeLists.txt b/plugins/hifiNeuron/CMakeLists.txt new file mode 100644 index 0000000000..9c512fc877 --- /dev/null +++ b/plugins/hifiNeuron/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# Created by Anthony Thibault on 2015/12/18 +# Copyright 2015 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# + +set(TARGET_NAME hifiNeuron) +setup_hifi_plugin(Script Qml Widgets) +link_hifi_libraries(shared controllers plugins input-plugins) +target_neuron() + diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp new file mode 100644 index 0000000000..a175ce8e06 --- /dev/null +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -0,0 +1,557 @@ +// +// NeuronPlugin.cpp +// input-plugins/src/input-plugins +// +// Created by Anthony Thibault on 12/18/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "NeuronPlugin.h" + +#include +#include +#include +#include +#include +#include +#include + +Q_DECLARE_LOGGING_CATEGORY(inputplugins) +Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") + +#define __OS_XUN__ 1 +#define BOOL int + +#ifdef HAVE_NEURON +#include +#endif + +const QString NeuronPlugin::NAME = "Neuron"; +const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; + +// indices of joints of the Neuron standard skeleton. +// This is 'almost' the same as the High Fidelity standard skeleton. +// It is missing a thumb joint. +enum NeuronJointIndex { + Hips = 0, + RightUpLeg, + RightLeg, + RightFoot, + LeftUpLeg, + LeftLeg, + LeftFoot, + Spine, + Spine1, + Spine2, + Spine3, + Neck, + Head, + RightShoulder, + RightArm, + RightForeArm, + RightHand, + RightHandThumb1, + RightHandThumb2, + RightHandThumb3, + RightInHandIndex, + RightHandIndex1, + RightHandIndex2, + RightHandIndex3, + RightInHandMiddle, + RightHandMiddle1, + RightHandMiddle2, + RightHandMiddle3, + RightInHandRing, + RightHandRing1, + RightHandRing2, + RightHandRing3, + RightInHandPinky, + RightHandPinky1, + RightHandPinky2, + RightHandPinky3, + LeftShoulder, + LeftArm, + LeftForeArm, + LeftHand, + LeftHandThumb1, + LeftHandThumb2, + LeftHandThumb3, + LeftInHandIndex, + LeftHandIndex1, + LeftHandIndex2, + LeftHandIndex3, + LeftInHandMiddle, + LeftHandMiddle1, + LeftHandMiddle2, + LeftHandMiddle3, + LeftInHandRing, + LeftHandRing1, + LeftHandRing2, + LeftHandRing3, + LeftInHandPinky, + LeftHandPinky1, + LeftHandPinky2, + LeftHandPinky3, + Size +}; + +// Almost a direct mapping except for LEFT_HAND_THUMB1 and RIGHT_HAND_THUMB1, +// which are not present in the Neuron standard skeleton. +static controller::StandardPoseChannel neuronJointIndexToPoseIndexMap[NeuronJointIndex::Size] = { + controller::HIPS, + controller::RIGHT_UP_LEG, + controller::RIGHT_LEG, + controller::RIGHT_FOOT, + controller::LEFT_UP_LEG, + controller::LEFT_LEG, + controller::LEFT_FOOT, + controller::SPINE, + controller::SPINE1, + controller::SPINE2, + controller::SPINE3, + controller::NECK, + controller::HEAD, + controller::RIGHT_SHOULDER, + controller::RIGHT_ARM, + controller::RIGHT_FORE_ARM, + controller::RIGHT_HAND, + controller::RIGHT_HAND_THUMB2, + controller::RIGHT_HAND_THUMB3, + controller::RIGHT_HAND_THUMB4, + controller::RIGHT_HAND_INDEX1, + controller::RIGHT_HAND_INDEX2, + controller::RIGHT_HAND_INDEX3, + controller::RIGHT_HAND_INDEX4, + controller::RIGHT_HAND_MIDDLE1, + controller::RIGHT_HAND_MIDDLE2, + controller::RIGHT_HAND_MIDDLE3, + controller::RIGHT_HAND_MIDDLE4, + controller::RIGHT_HAND_RING1, + controller::RIGHT_HAND_RING2, + controller::RIGHT_HAND_RING3, + controller::RIGHT_HAND_RING4, + controller::RIGHT_HAND_PINKY1, + controller::RIGHT_HAND_PINKY2, + controller::RIGHT_HAND_PINKY3, + controller::RIGHT_HAND_PINKY4, + controller::LEFT_SHOULDER, + controller::LEFT_ARM, + controller::LEFT_FORE_ARM, + controller::LEFT_HAND, + controller::LEFT_HAND_THUMB2, + controller::LEFT_HAND_THUMB3, + controller::LEFT_HAND_THUMB4, + controller::LEFT_HAND_INDEX1, + controller::LEFT_HAND_INDEX2, + controller::LEFT_HAND_INDEX3, + controller::LEFT_HAND_INDEX4, + controller::LEFT_HAND_MIDDLE1, + controller::LEFT_HAND_MIDDLE2, + controller::LEFT_HAND_MIDDLE3, + controller::LEFT_HAND_MIDDLE4, + controller::LEFT_HAND_RING1, + controller::LEFT_HAND_RING2, + controller::LEFT_HAND_RING3, + controller::LEFT_HAND_RING4, + controller::LEFT_HAND_PINKY1, + controller::LEFT_HAND_PINKY2, + controller::LEFT_HAND_PINKY3, + controller::LEFT_HAND_PINKY4 +}; + +// in rig frame +static glm::vec3 rightHandThumb1DefaultAbsTranslation(-2.155500650405884, -0.7610001564025879, 2.685631036758423); +static glm::vec3 leftHandThumb1DefaultAbsTranslation(2.1555817127227783, -0.7603635787963867, 2.6856393814086914); + +// default translations (cm) +static glm::vec3 neuronJointTranslations[NeuronJointIndex::Size] = { + {131.901, 95.6602, -27.9815}, + {-9.55907, -1.58772, 0.0760284}, + {0.0144232, -41.4683, -0.105322}, + {1.59348, -41.5875, -0.557237}, + {9.72077, -1.68926, -0.280643}, + {0.0886684, -43.1586, -0.0111596}, + {-2.98473, -44.0517, 0.0694456}, + {0.110967, 16.3959, 0.140463}, + {0.0500451, 10.0238, 0.0731921}, + {0.061568, 10.4352, 0.0583075}, + {0.0500606, 10.0217, 0.0711083}, + {0.0317731, 10.7176, 0.0779325}, + {-0.0204253, 9.71067, 0.131734}, + {-3.24245, 7.13584, 0.185638}, + {-13.0885, -0.0877601, 0.176065}, + {-27.2674, 0.0688724, 0.0272146}, + {-26.7673, 0.0301916, 0.0102847}, + {-2.56017, 0.195537, 3.20968}, + {-3.78796, 0, 0}, + {-2.63141, 0, 0}, + {-3.31579, 0.522947, 2.03495}, + {-5.36589, -0.0939789, 1.02771}, + {-3.72278, 0, 0}, + {-2.11074, 0, 0}, + {-3.47874, 0.532042, 0.778358}, + {-5.32194, -0.0864, 0.322863}, + {-4.06232, 0, 0}, + {-2.54653, 0, 0}, + {-3.46131, 0.553263, -0.132632}, + {-4.76716, -0.0227368, -0.492632}, + {-3.54073, 0, 0}, + {-2.45634, 0, 0}, + {-3.25137, 0.482779, -1.23613}, + {-4.25937, -0.0227368, -1.12168}, + {-2.83528, 0, 0}, + {-1.79166, 0, 0}, + {3.25624, 7.13148, -0.131575}, + {13.149, -0.052598, -0.125076}, + {27.2903, 0.00282644, -0.0181535}, + {26.6602, 0.000969969, -0.0487599}, + {2.56017, 0.195537, 3.20968}, + {3.78796, 0, 0}, + {2.63141, 0, 0}, + {3.31579, 0.522947, 2.03495}, + {5.36589, -0.0939789, 1.02771}, + {3.72278, 0, 0}, + {2.11074, 0, 0}, + {3.47874, 0.532042, 0.778358}, + {5.32194, -0.0864, 0.322863}, + {4.06232, 0, 0}, + {2.54653, 0, 0}, + {3.46131, 0.553263, -0.132632}, + {4.76716, -0.0227368, -0.492632}, + {3.54073, 0, 0}, + {2.45634, 0, 0}, + {3.25137, 0.482779, -1.23613}, + {4.25937, -0.0227368, -1.12168}, + {2.83528, 0, 0}, + {1.79166, 0, 0} +}; + +static controller::StandardPoseChannel neuronJointIndexToPoseIndex(NeuronJointIndex i) { + assert(i >= 0 && i < NeuronJointIndex::Size); + if (i >= 0 && i < NeuronJointIndex::Size) { + return neuronJointIndexToPoseIndexMap[i]; + } else { + return (controller::StandardPoseChannel)0; // not sure what to do here, but don't crash! + } +} + +static const char* controllerJointName(controller::StandardPoseChannel i) { + switch (i) { + case controller::HIPS: return "Hips"; + case controller::RIGHT_UP_LEG: return "RightUpLeg"; + case controller::RIGHT_LEG: return "RightLeg"; + case controller::RIGHT_FOOT: return "RightFoot"; + case controller::LEFT_UP_LEG: return "LeftUpLeg"; + case controller::LEFT_LEG: return "LeftLeg"; + case controller::LEFT_FOOT: return "LeftFoot"; + case controller::SPINE: return "Spine"; + case controller::SPINE1: return "Spine1"; + case controller::SPINE2: return "Spine2"; + case controller::SPINE3: return "Spine3"; + case controller::NECK: return "Neck"; + case controller::HEAD: return "Head"; + case controller::RIGHT_SHOULDER: return "RightShoulder"; + case controller::RIGHT_ARM: return "RightArm"; + case controller::RIGHT_FORE_ARM: return "RightForeArm"; + case controller::RIGHT_HAND: return "RightHand"; + case controller::RIGHT_HAND_THUMB1: return "RightHandThumb1"; + case controller::RIGHT_HAND_THUMB2: return "RightHandThumb2"; + case controller::RIGHT_HAND_THUMB3: return "RightHandThumb3"; + case controller::RIGHT_HAND_THUMB4: return "RightHandThumb4"; + case controller::RIGHT_HAND_INDEX1: return "RightHandIndex1"; + case controller::RIGHT_HAND_INDEX2: return "RightHandIndex2"; + case controller::RIGHT_HAND_INDEX3: return "RightHandIndex3"; + case controller::RIGHT_HAND_INDEX4: return "RightHandIndex4"; + case controller::RIGHT_HAND_MIDDLE1: return "RightHandMiddle1"; + case controller::RIGHT_HAND_MIDDLE2: return "RightHandMiddle2"; + case controller::RIGHT_HAND_MIDDLE3: return "RightHandMiddle3"; + case controller::RIGHT_HAND_MIDDLE4: return "RightHandMiddle4"; + case controller::RIGHT_HAND_RING1: return "RightHandRing1"; + case controller::RIGHT_HAND_RING2: return "RightHandRing2"; + case controller::RIGHT_HAND_RING3: return "RightHandRing3"; + case controller::RIGHT_HAND_RING4: return "RightHandRing4"; + case controller::RIGHT_HAND_PINKY1: return "RightHandPinky1"; + case controller::RIGHT_HAND_PINKY2: return "RightHandPinky2"; + case controller::RIGHT_HAND_PINKY3: return "RightHandPinky3"; + case controller::RIGHT_HAND_PINKY4: return "RightHandPinky4"; + case controller::LEFT_SHOULDER: return "LeftShoulder"; + case controller::LEFT_ARM: return "LeftArm"; + case controller::LEFT_FORE_ARM: return "LeftForeArm"; + case controller::LEFT_HAND: return "LeftHand"; + case controller::LEFT_HAND_THUMB1: return "LeftHandThumb1"; + case controller::LEFT_HAND_THUMB2: return "LeftHandThumb2"; + case controller::LEFT_HAND_THUMB3: return "LeftHandThumb3"; + case controller::LEFT_HAND_THUMB4: return "LeftHandThumb4"; + case controller::LEFT_HAND_INDEX1: return "LeftHandIndex1"; + case controller::LEFT_HAND_INDEX2: return "LeftHandIndex2"; + case controller::LEFT_HAND_INDEX3: return "LeftHandIndex3"; + case controller::LEFT_HAND_INDEX4: return "LeftHandIndex4"; + case controller::LEFT_HAND_MIDDLE1: return "LeftHandMiddle1"; + case controller::LEFT_HAND_MIDDLE2: return "LeftHandMiddle2"; + case controller::LEFT_HAND_MIDDLE3: return "LeftHandMiddle3"; + case controller::LEFT_HAND_MIDDLE4: return "LeftHandMiddle4"; + case controller::LEFT_HAND_RING1: return "LeftHandRing1"; + case controller::LEFT_HAND_RING2: return "LeftHandRing2"; + case controller::LEFT_HAND_RING3: return "LeftHandRing3"; + case controller::LEFT_HAND_RING4: return "LeftHandRing4"; + case controller::LEFT_HAND_PINKY1: return "LeftHandPinky1"; + case controller::LEFT_HAND_PINKY2: return "LeftHandPinky2"; + case controller::LEFT_HAND_PINKY3: return "LeftHandPinky3"; + case controller::LEFT_HAND_PINKY4: return "LeftHandPinky4"; + default: return "???"; + } +} + +// convert between YXZ neuron euler angles in degrees to quaternion +// this is the default setting in the Axis Neuron server. +static quat eulerToQuat(vec3 euler) { + // euler.x and euler.y are swaped, WTF. + glm::vec3 e = glm::vec3(euler.y, euler.x, euler.z) * RADIANS_PER_DEGREE; + return (glm::angleAxis(e.y, Vectors::UNIT_Y) * + glm::angleAxis(e.x, Vectors::UNIT_X) * + glm::angleAxis(e.z, Vectors::UNIT_Z)); +} + +#ifdef HAVE_NEURON + +// +// neuronDataReader SDK callback functions +// + +// NOTE: must be thread-safe +void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx* header, float* data) { + + auto neuronPlugin = reinterpret_cast(context); + + // version 1.0 + if (header->DataVersion.Major == 1 && header->DataVersion.Minor == 0) { + + // skip reference joint if present + if (header->WithReference && header->WithDisp) { + data += 6; + } else if (header->WithReference && !header->WithDisp) { + data += 3; + } + + if (header->WithDisp) { + // enter mutex + std::lock_guard guard(neuronPlugin->_jointsMutex); + + // + // Data is 6 floats per joint: 3 position values, 3 rotation euler angles (degrees) + // + + // resize vector if necessary + const size_t NUM_FLOATS_PER_JOINT = 6; + const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; + if (neuronPlugin->_joints.size() != NUM_JOINTS) { + neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + } + + assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); + + // copy the data + memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); + + } else { + qCWarning(inputplugins) << "NeuronPlugin: unsuported binary format, please enable displacements"; + + // enter mutex + std::lock_guard guard(neuronPlugin->_jointsMutex); + + if (neuronPlugin->_joints.size() != NeuronJointIndex::Size) { + neuronPlugin->_joints.resize(NeuronJointIndex::Size, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + } + + for (int i = 0; i < NeuronJointIndex::Size; i++) { + neuronPlugin->_joints[i].euler = glm::vec3(); + neuronPlugin->_joints[i].pos = neuronJointTranslations[i]; + } + } + } else { + static bool ONCE = false; + if (!ONCE) { + qCCritical(inputplugins) << "NeuronPlugin: bad frame version number, expected 1.0"; + ONCE = true; + } + } +} + +// I can't even get the SDK to send me a callback. +// BRCommandFetchAvatarDataFromServer & BRRegisterAutoSyncParmeter [sic] don't seem to work. +// So this is totally untested. +// NOTE: must be thread-safe +static void CommandDataReceivedCallback(void* context, SOCKET_REF sender, CommandPack* pack, void* data) { + + DATA_VER version; + version._VersionMask = pack->DataVersion; + if (version.Major == 1 && version.Minor == 0) { + const char* str = "Unknown"; + switch (pack->CommandId) { + case Cmd_BoneSize: // Id can be used to request bone size from server or register avatar name command. + str = "BoneSize"; + break; + case Cmd_AvatarName: // Id can be used to request avatar name from server or register avatar name command. + str = "AvatarName"; + break; + case Cmd_FaceDirection: // Id used to request face direction from server + str = "FaceDirection"; + break; + case Cmd_DataFrequency: // Id can be used to request data frequency from server or register data frequency command. + str = "DataFrequency"; + break; + case Cmd_BvhInheritanceTxt: // Id can be used to request bvh header txt from server or register bvh header txt command. + str = "BvhInheritanceTxt"; + break; + case Cmd_AvatarCount: // Id can be used to request avatar count from server or register avatar count command. + str = "AvatarCount"; + break; + case Cmd_CombinationMode: // Id can be used to request combination mode from server or register combination mode command. + str = "CombinationMode"; + break; + case Cmd_RegisterEvent: // Id can be used to register event. + str = "RegisterEvent"; + break; + case Cmd_UnRegisterEvent: // Id can be used to unregister event. + str = "UnRegisterEvent"; + break; + } + qCDebug(inputplugins) << "NeuronPlugin: command data received CommandID = " << str; + } else { + static bool ONCE = false; + if (!ONCE) { + qCCritical(inputplugins) << "NeuronPlugin: bad command version number, expected 1.0"; + ONCE = true; + } + } +} + +// NOTE: must be thread-safe +static void SocketStatusChangedCallback(void* context, SOCKET_REF sender, SocketStatus status, char* message) { + // just dump to log, later we might want to pop up a connection lost dialog or attempt to reconnect. + qCDebug(inputplugins) << "NeuronPlugin: socket status = " << message; +} + +#endif // #ifdef HAVE_NEURON + +// +// NeuronPlugin +// + +bool NeuronPlugin::isSupported() const { +#ifdef HAVE_NEURON + // Because it's a client/server network architecture, we can't tell + // if the neuron is actually connected until we connect to the server. + return true; +#else + return false; +#endif +} + +void NeuronPlugin::activate() { +#ifdef HAVE_NEURON + InputPlugin::activate(); + + // register with userInputMapper + auto userInputMapper = DependencyManager::get(); + userInputMapper->registerDevice(_inputDevice); + + // register c-style callbacks + BRRegisterFrameDataCallback((void*)this, FrameDataReceivedCallback); + BRRegisterCommandDataCallback((void*)this, CommandDataReceivedCallback); + BRRegisterSocketStatusCallback((void*)this, SocketStatusChangedCallback); + + // TODO: Pull these from prefs dialog? + // localhost is fine for now. + _serverAddress = "localhost"; + _serverPort = 7001; // default port for TCP Axis Neuron server. + + _socketRef = BRConnectTo((char*)_serverAddress.c_str(), _serverPort); + if (!_socketRef) { + // error + qCCritical(inputplugins) << "NeuronPlugin: error connecting to " << _serverAddress.c_str() << ":" << _serverPort << ", error = " << BRGetLastErrorMessage(); + } else { + qCDebug(inputplugins) << "NeuronPlugin: success connecting to " << _serverAddress.c_str() << ":" << _serverPort; + + BRRegisterAutoSyncParmeter(_socketRef, Cmd_CombinationMode); + } +#endif +} + +void NeuronPlugin::deactivate() { +#ifdef HAVE_NEURON + // unregister from userInputMapper + if (_inputDevice->_deviceID != controller::Input::INVALID_DEVICE) { + auto userInputMapper = DependencyManager::get(); + userInputMapper->removeDevice(_inputDevice->_deviceID); + } + + if (_socketRef) { + BRUnregisterAutoSyncParmeter(_socketRef, Cmd_CombinationMode); + BRCloseSocket(_socketRef); + } + + InputPlugin::deactivate(); +#endif +} + +void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { + std::vector joints; + { + // lock and copy + std::lock_guard guard(_jointsMutex); + joints = _joints; + } + _inputDevice->update(deltaTime, joints, _prevJoints); + _prevJoints = joints; +} + +void NeuronPlugin::saveSettings() const { + InputPlugin::saveSettings(); +} + +void NeuronPlugin::loadSettings() { + InputPlugin::loadSettings(); +} + +// +// InputDevice +// + +controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const { + static controller::Input::NamedVector availableInputs; + if (availableInputs.size() == 0) { + for (int i = 0; i < controller::NUM_STANDARD_POSES; i++) { + auto channel = static_cast(i); + availableInputs.push_back(makePair(channel, controllerJointName(channel))); + } + }; + return availableInputs; +} + +QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { + static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/neuron.json"; + return MAPPING_JSON; +} + +void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector& joints, const std::vector& prevJoints) { + for (size_t i = 0; i < joints.size(); i++) { + glm::vec3 linearVel, angularVel; + glm::vec3 pos = joints[i].pos; + glm::quat rot = eulerToQuat(joints[i].euler); + if (i < prevJoints.size()) { + linearVel = (pos - (prevJoints[i].pos * METERS_PER_CENTIMETER)) / deltaTime; // m/s + // quat log imaginary part points along the axis of rotation, with length of one half the angle of rotation. + glm::quat d = glm::log(rot * glm::inverse(eulerToQuat(prevJoints[i].euler))); + angularVel = glm::vec3(d.x, d.y, d.z) / (0.5f * deltaTime); // radians/s + } + int poseIndex = neuronJointIndexToPoseIndex((NeuronJointIndex)i); + _poseStateMap[poseIndex] = controller::Pose(pos, rot, linearVel, angularVel); + } + + _poseStateMap[controller::RIGHT_HAND_THUMB1] = controller::Pose(rightHandThumb1DefaultAbsTranslation, glm::quat(), glm::vec3(), glm::vec3()); + _poseStateMap[controller::LEFT_HAND_THUMB1] = controller::Pose(leftHandThumb1DefaultAbsTranslation, glm::quat(), glm::vec3(), glm::vec3()); +} diff --git a/plugins/hifiNeuron/src/NeuronPlugin.h b/plugins/hifiNeuron/src/NeuronPlugin.h new file mode 100644 index 0000000000..c85a5dd383 --- /dev/null +++ b/plugins/hifiNeuron/src/NeuronPlugin.h @@ -0,0 +1,85 @@ +// +// NeuronPlugin.h +// input-plugins/src/input-plugins +// +// Created by Anthony Thibault on 12/18/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_NeuronPlugin_h +#define hifi_NeuronPlugin_h + +#include +#include +#include + +struct _BvhDataHeaderEx; +void FrameDataReceivedCallback(void* context, void* sender, _BvhDataHeaderEx* header, float* data); + +// Handles interaction with the Neuron SDK +class NeuronPlugin : public InputPlugin { + Q_OBJECT +public: + friend void FrameDataReceivedCallback(void* context, void* sender, _BvhDataHeaderEx* header, float* data); + + // Plugin functions + virtual bool isSupported() const override; + virtual bool isJointController() const override { return true; } + const QString& getName() const override { return NAME; } + const QString& getID() const override { return NEURON_ID_STRING; } + + virtual void activate() override; + virtual void deactivate() override; + + virtual void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); } + virtual void pluginUpdate(float deltaTime, bool jointsCaptured) override; + + virtual void saveSettings() const override; + virtual void loadSettings() override; + +protected: + + struct NeuronJoint { + glm::vec3 pos; + glm::vec3 euler; + }; + + class InputDevice : public controller::InputDevice { + public: + friend class NeuronPlugin; + + InputDevice() : controller::InputDevice("Neuron") {} + + // Device functions + virtual controller::Input::NamedVector getAvailableInputs() const override; + virtual QString getDefaultMappingConfig() const override; + virtual void update(float deltaTime, bool jointsCaptured) override {}; + virtual void focusOutEvent() override {}; + + void update(float deltaTime, const std::vector& joints, const std::vector& prevJoints); + }; + + std::shared_ptr _inputDevice { std::make_shared() }; + + static const QString NAME; + static const QString NEURON_ID_STRING; + + std::string _serverAddress; + int _serverPort; + void* _socketRef; + + // used to guard multi-threaded access to _joints + std::mutex _jointsMutex; + + // copy of data directly from the NeuronDataReader SDK + std::vector _joints; + + // one frame old copy of _joints, used to caluclate angular and linear velocity. + std::vector _prevJoints; +}; + +#endif // hifi_NeuronPlugin_h + diff --git a/plugins/hifiNeuron/src/NeuronProvider.cpp b/plugins/hifiNeuron/src/NeuronProvider.cpp new file mode 100644 index 0000000000..b171c5150d --- /dev/null +++ b/plugins/hifiNeuron/src/NeuronProvider.cpp @@ -0,0 +1,45 @@ +// +// Created by Anthony Thibault on 2015/12/18 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include +#include + +#include +#include + +#include "NeuronPlugin.h" + +class NeuronProvider : public QObject, public InputProvider +{ + Q_OBJECT + Q_PLUGIN_METADATA(IID InputProvider_iid FILE "plugin.json") + Q_INTERFACES(InputProvider) + +public: + NeuronProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~NeuronProvider() {} + + virtual InputPluginList getInputPlugins() override { + static std::once_flag once; + std::call_once(once, [&] { + InputPluginPointer plugin(new NeuronPlugin()); + if (plugin->isSupported()) { + _inputPlugins.push_back(plugin); + } + }); + return _inputPlugins; + } + +private: + InputPluginList _inputPlugins; +}; + +#include "NeuronProvider.moc" diff --git a/plugins/hifiNeuron/src/plugin.json b/plugins/hifiNeuron/src/plugin.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/plugins/hifiNeuron/src/plugin.json @@ -0,0 +1 @@ +{}