From dcde640acdde332a861b6d8d9117855bf1685360 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 18 Dec 2015 19:13:59 -0800 Subject: [PATCH 01/13] Stub Perception Neuron input plugin --- interface/resources/controllers/neuron.json | 7 ++ plugins/neuron/CMakeLists.txt | 13 ++++ plugins/neuron/src/NeuronPlugin.cpp | 75 +++++++++++++++++++++ plugins/neuron/src/NeuronPlugin.h | 57 ++++++++++++++++ plugins/neuron/src/NeuronProvider.cpp | 45 +++++++++++++ plugins/neuron/src/plugin.json | 1 + 6 files changed, 198 insertions(+) create mode 100644 interface/resources/controllers/neuron.json create mode 100644 plugins/neuron/CMakeLists.txt create mode 100644 plugins/neuron/src/NeuronPlugin.cpp create mode 100644 plugins/neuron/src/NeuronPlugin.h create mode 100644 plugins/neuron/src/NeuronProvider.cpp create mode 100644 plugins/neuron/src/plugin.json diff --git a/interface/resources/controllers/neuron.json b/interface/resources/controllers/neuron.json new file mode 100644 index 0000000000..2d61f80c35 --- /dev/null +++ b/interface/resources/controllers/neuron.json @@ -0,0 +1,7 @@ +{ + "name": "Neuron to Standard", + "channels": [ + { "from": "Hydra.LeftHand", "to": "Standard.LeftHand" }, + { "from": "Hydra.RightHand", "to": "Standard.RightHand" } + ] +} diff --git a/plugins/neuron/CMakeLists.txt b/plugins/neuron/CMakeLists.txt new file mode 100644 index 0000000000..b86d310ab7 --- /dev/null +++ b/plugins/neuron/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# Created by Anthony Thibault on 2015/12/18 +# Copyright 2015 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# + +set(TARGET_NAME neuron) +setup_hifi_plugin(Script Qml Widgets) +link_hifi_libraries(shared controllers plugins input-plugins) +# target_neuron() + diff --git a/plugins/neuron/src/NeuronPlugin.cpp b/plugins/neuron/src/NeuronPlugin.cpp new file mode 100644 index 0000000000..735b81a1ef --- /dev/null +++ b/plugins/neuron/src/NeuronPlugin.cpp @@ -0,0 +1,75 @@ +// +// NeuronPlugin.h +// input-plugins/src/input-plugins +// +// Created by Anthony Thibault on 12/18/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "NeuronPlugin.h" + +#include +#include + +Q_DECLARE_LOGGING_CATEGORY(inputplugins) +Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") + +const QString NeuronPlugin::NAME = "Neuron"; +const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; + +bool NeuronPlugin::isSupported() const { + // TODO: + return true; +} + +void NeuronPlugin::activate() { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::activate"; +} + +void NeuronPlugin::deactivate() { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::deactivate"; +} + +void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::pluginUpdate"; +} + +void NeuronPlugin::saveSettings() const { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::saveSettings"; +} + +void NeuronPlugin::loadSettings() { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::loadSettings"; +} + +controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const { + // TODO: + static const controller::Input::NamedVector availableInputs { + makePair(controller::LEFT_HAND, "LeftHand"), + makePair(controller::RIGHT_HAND, "RightHand") + }; + return availableInputs; +} + +QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { + static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/neuron.json"; + return MAPPING_JSON; +} + +void NeuronPlugin::InputDevice::update(float deltaTime, bool jointsCaptured) { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::InputDevice::update"; +} + +void NeuronPlugin::InputDevice::focusOutEvent() { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::InputDevice::focusOutEvent"; +} diff --git a/plugins/neuron/src/NeuronPlugin.h b/plugins/neuron/src/NeuronPlugin.h new file mode 100644 index 0000000000..59e0f0a393 --- /dev/null +++ b/plugins/neuron/src/NeuronPlugin.h @@ -0,0 +1,57 @@ +// +// NeuronPlugin.h +// input-plugins/src/input-plugins +// +// Created by Anthony Thibault on 12/18/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_NeuronPlugin_h +#define hifi_NeuronPlugin_h + +#include +#include +#include + +// Handles interaction with the Neuron SDK +class NeuronPlugin : public InputPlugin { + Q_OBJECT +public: + // Plugin functions + virtual bool isSupported() const override; + virtual bool isJointController() const override { return true; } + const QString& getName() const override { return NAME; } + const QString& getID() const override { return NEURON_ID_STRING; } + + virtual void activate() override; + virtual void deactivate() override; + + virtual void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); } + virtual void pluginUpdate(float deltaTime, bool jointsCaptured) override; + + virtual void saveSettings() const override; + virtual void loadSettings() override; + +private: + class InputDevice : public controller::InputDevice { + public: + InputDevice() : controller::InputDevice("Neuron") {} + + // Device functions + virtual controller::Input::NamedVector getAvailableInputs() const override; + virtual QString getDefaultMappingConfig() const override; + virtual void update(float deltaTime, bool jointsCaptured) override; + virtual void focusOutEvent() override; + }; + + std::shared_ptr _inputDevice { std::make_shared() }; + + static const QString NAME; + static const QString NEURON_ID_STRING; +}; + +#endif // hifi_NeuronPlugin_h + diff --git a/plugins/neuron/src/NeuronProvider.cpp b/plugins/neuron/src/NeuronProvider.cpp new file mode 100644 index 0000000000..b171c5150d --- /dev/null +++ b/plugins/neuron/src/NeuronProvider.cpp @@ -0,0 +1,45 @@ +// +// Created by Anthony Thibault on 2015/12/18 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include +#include + +#include +#include + +#include "NeuronPlugin.h" + +class NeuronProvider : public QObject, public InputProvider +{ + Q_OBJECT + Q_PLUGIN_METADATA(IID InputProvider_iid FILE "plugin.json") + Q_INTERFACES(InputProvider) + +public: + NeuronProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~NeuronProvider() {} + + virtual InputPluginList getInputPlugins() override { + static std::once_flag once; + std::call_once(once, [&] { + InputPluginPointer plugin(new NeuronPlugin()); + if (plugin->isSupported()) { + _inputPlugins.push_back(plugin); + } + }); + return _inputPlugins; + } + +private: + InputPluginList _inputPlugins; +}; + +#include "NeuronProvider.moc" diff --git a/plugins/neuron/src/plugin.json b/plugins/neuron/src/plugin.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/plugins/neuron/src/plugin.json @@ -0,0 +1 @@ +{} From 0459479c2b1a8130beecee5771d40aac490d1a9f Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Mon, 21 Dec 2015 18:30:15 -0800 Subject: [PATCH 02/13] NeuronPlugin: Added external project for Neuron SDK Now builds on windows with actual Neuron SDK. Can to TCP server on localhost, and receive joint data. Will debug draw joint 6, (left foot?) --- cmake/externals/neuron/CMakeLists.txt | 49 ++++ cmake/macros/TargetNeuron.cmake | 14 ++ cmake/modules/FindNeuron.cmake | 28 +++ interface/CMakeLists.txt | 4 +- plugins/{neuron => hifiNeuron}/CMakeLists.txt | 4 +- plugins/hifiNeuron/src/NeuronPlugin.cpp | 222 ++++++++++++++++++ .../{neuron => hifiNeuron}/src/NeuronPlugin.h | 19 +- .../src/NeuronProvider.cpp | 0 .../{neuron => hifiNeuron}/src/plugin.json | 0 plugins/neuron/src/NeuronPlugin.cpp | 75 ------ 10 files changed, 336 insertions(+), 79 deletions(-) create mode 100644 cmake/externals/neuron/CMakeLists.txt create mode 100644 cmake/macros/TargetNeuron.cmake create mode 100644 cmake/modules/FindNeuron.cmake rename plugins/{neuron => hifiNeuron}/CMakeLists.txt (88%) create mode 100644 plugins/hifiNeuron/src/NeuronPlugin.cpp rename plugins/{neuron => hifiNeuron}/src/NeuronPlugin.h (79%) rename plugins/{neuron => hifiNeuron}/src/NeuronProvider.cpp (100%) rename plugins/{neuron => hifiNeuron}/src/plugin.json (100%) delete mode 100644 plugins/neuron/src/NeuronPlugin.cpp diff --git a/cmake/externals/neuron/CMakeLists.txt b/cmake/externals/neuron/CMakeLists.txt new file mode 100644 index 0000000000..324b3fb917 --- /dev/null +++ b/cmake/externals/neuron/CMakeLists.txt @@ -0,0 +1,49 @@ +include(ExternalProject) + +set(EXTERNAL_NAME neuron) + +string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) + +set(NEURON_URL "https://s3.amazonaws.com/hifi-public/dependencies/neuron_datareader_b.12.zip") +set(NEURON_URL_MD5 "0ab54ca04c9cc8094e0fa046c226e574") + +ExternalProject_Add(${EXTERNAL_NAME} + URL ${NEURON_URL} + URL_MD5 ${NEURON_URL_MD5} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + LOG_DOWNLOAD 1) + +ExternalProject_Get_Property(${EXTERNAL_NAME} SOURCE_DIR) + +set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals") + +# set include dir +if(WIN32) + set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS "${SOURCE_DIR}/NeuronDataReader_Windows/include" CACHE TYPE INTERNAL) +elseif(APPLE) + set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS "${SOURCE_DIR}/NeuronDataReader_Mac/include" CACHE TYPE INTERNAL) +else() + # Unsupported +endif() + +if(WIN32) + + if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + set(ARCH_DIR "x64") + else() + set(ARCH_DIR "x86") + endif() + + set(${EXTERNAL_NAME_UPPER}_LIB_PATH "${SOURCE_DIR}/NeuronDataReader_Windows/lib/${ARCH_DIR}") + set(${EXTERNAL_NAME_UPPER}_LIBRARY_RELEASE "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.lib" CACHE TYPE INTERNAL) + set(${EXTERNAL_NAME_UPPER}_LIBRARIES "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.lib" CACHE TYPE INTERNAL) + + add_paths_to_fixup_libs("${${EXTERNAL_NAME_UPPER}_LIB_PATH}") +elseif(APPLE) + # TODO +else() + # UNSUPPORTED +endif() + diff --git a/cmake/macros/TargetNeuron.cmake b/cmake/macros/TargetNeuron.cmake new file mode 100644 index 0000000000..01891ef525 --- /dev/null +++ b/cmake/macros/TargetNeuron.cmake @@ -0,0 +1,14 @@ +# +# Copyright 2015 High Fidelity, Inc. +# Created by Anthony J. Thibault on 2015/12/21 +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# +macro(TARGET_NEURON) + add_dependency_external_projects(neuron) + find_package(Neuron REQUIRED) + target_include_directories(${TARGET_NAME} PRIVATE ${NEURON_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${NEURON_LIBRARIES}) + add_definitions(-DHAVE_NEURON) +endmacro() diff --git a/cmake/modules/FindNeuron.cmake b/cmake/modules/FindNeuron.cmake new file mode 100644 index 0000000000..6a93b3a016 --- /dev/null +++ b/cmake/modules/FindNeuron.cmake @@ -0,0 +1,28 @@ +# +# FindNeuron.cmake +# +# Try to find the Perception Neuron SDK +# +# You must provide a NEURON_ROOT_DIR which contains lib and include directories +# +# Once done this will define +# +# NEURON_FOUND - system found Neuron SDK +# NEURON_INCLUDE_DIRS - the Neuron SDK include directory +# NEURON_LIBRARIES - Link this to use Neuron +# +# Created on 12/21/2015 by Anthony J. Thibault +# Copyright 2015 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# + +include(SelectLibraryConfigurations) +select_library_configurations(NEURON) + +set(NEURON_REQUIREMENTS NEURON_INCLUDE_DIRS NEURON_LIBRARIES) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Neuron DEFAULT_MSG NEURON_INCLUDE_DIRS NEURON_LIBRARIES) +mark_as_advanced(NEURON_LIBRARIES NEURON_INCLUDE_DIRS NEURON_SEARCH_DIRS) + diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 1d9557a835..5d96b95624 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -109,7 +109,9 @@ add_dependency_external_projects(sdl2) if (WIN32) add_dependency_external_projects(OpenVR) endif() - +if(WIN32 OR APPLE) + add_dependency_external_projects(neuron) +endif() # disable /OPT:REF and /OPT:ICF for the Debug builds # This will prevent the following linker warnings diff --git a/plugins/neuron/CMakeLists.txt b/plugins/hifiNeuron/CMakeLists.txt similarity index 88% rename from plugins/neuron/CMakeLists.txt rename to plugins/hifiNeuron/CMakeLists.txt index b86d310ab7..9c512fc877 100644 --- a/plugins/neuron/CMakeLists.txt +++ b/plugins/hifiNeuron/CMakeLists.txt @@ -6,8 +6,8 @@ # See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html # -set(TARGET_NAME neuron) +set(TARGET_NAME hifiNeuron) setup_hifi_plugin(Script Qml Widgets) link_hifi_libraries(shared controllers plugins input-plugins) -# target_neuron() +target_neuron() diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp new file mode 100644 index 0000000000..c3f764da05 --- /dev/null +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -0,0 +1,222 @@ +// +// NeuronPlugin.h +// input-plugins/src/input-plugins +// +// Created by Anthony Thibault on 12/18/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "NeuronPlugin.h" + +#include +#include +#include +#include +#include + +Q_DECLARE_LOGGING_CATEGORY(inputplugins) +Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") + +#define __OS_XUN__ 1 +#define BOOL int +#include + +const QString NeuronPlugin::NAME = "Neuron"; +const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; + +enum JointIndex { + HipsPosition = 0, + Hips, + RightUpLeg, + RightLeg, + RightFoot, + LeftUpLeg, + LeftLeg, + LeftFoot, + Spine, + Spine1, + Spine2, + Spine3, + Neck, + Head, + RightShoulder, + RightArm, + RightForeArm, + RightHand, + RightHandThumb1, + RightHandThumb2, + RightHandThumb3, + RightInHandIndex, + RightHandIndex1, + RightHandIndex2, + RightHandIndex3, + RightInHandMiddle, + RightHandMiddle1, + RightHandMiddle2, + RightHandMiddle3, + RightInHandRing, + RightHandRing1, + RightHandRing2, + RightHandRing3, + RightInHandPinky, + RightHandPinky1, + RightHandPinky2, + RightHandPinky3, + LeftShoulder, + LeftArm, + LeftForeArm, + LeftHand, + LeftHandThumb1, + LeftHandThumb2, + LeftHandThumb3, + LeftInHandIndex, + LeftHandIndex1, + LeftHandIndex2, + LeftHandIndex3, + LeftInHandMiddle, + LeftHandMiddle1, + LeftHandMiddle2, + LeftHandMiddle3, + LeftInHandRing, + LeftHandRing1, + LeftHandRing2, + LeftHandRing3, + LeftInHandPinky, + LeftHandPinky1, + LeftHandPinky2, + LeftHandPinky3 +}; + +bool NeuronPlugin::isSupported() const { + // Because it's a client/server network architecture, we can't tell + // if the neuron is actually connected until we connect to the server. + return true; +} + +// NOTE: must be thread-safe +void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx* header, float* data) { + qCDebug(inputplugins) << "NeuronPlugin: received frame data, DataCount = " << header->DataCount; + + auto neuronPlugin = reinterpret_cast(context); + std::lock_guard guard(neuronPlugin->_jointsMutex); + + // Data is 6 floats: 3 position values, 3 rotation euler angles (degrees) + + // resize vector if necessary + const size_t NUM_FLOATS_PER_JOINT = 6; + const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; + if (neuronPlugin->_joints.size() != NUM_JOINTS) { + neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + } + + assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); + + // copy the data + memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); +} + +// NOTE: must be thread-safe +static void CommandDataReceivedCallback(void* context, SOCKET_REF sender, CommandPack* pack, void* data) { + +} + +// NOTE: must be thread-safe +static void SocketStatusChangedCallback(void* context, SOCKET_REF sender, SocketStatus status, char* message) { + qCDebug(inputplugins) << "NeuronPlugin: socket status = " << message; +} + +void NeuronPlugin::activate() { + InputPlugin::activate(); + qCDebug(inputplugins) << "NeuronPlugin::activate"; + + // register c-style callbacks + BRRegisterFrameDataCallback((void*)this, FrameDataReceivedCallback); + BRRegisterCommandDataCallback((void*)this, CommandDataReceivedCallback); + BRRegisterSocketStatusCallback((void*)this, SocketStatusChangedCallback); + + // TODO: pull these from prefs! + _serverAddress = "localhost"; + _serverPort = 7001; + _socketRef = BRConnectTo((char*)_serverAddress.c_str(), _serverPort); + if (!_socketRef) { + // error + qCCritical(inputplugins) << "NeuronPlugin: error connecting to " << _serverAddress.c_str() << ":" << _serverPort << "error = " << BRGetLastErrorMessage(); + } + qCDebug(inputplugins) << "NeuronPlugin: success connecting to " << _serverAddress.c_str() << ":" << _serverPort; +} + +void NeuronPlugin::deactivate() { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::deactivate"; + + if (_socketRef) { + BRCloseSocket(_socketRef); + } + InputPlugin::deactivate(); +} + +// convert between euler in degrees to quaternion +static quat eulerToQuat(vec3 euler) { + return (glm::angleAxis(euler.y * RADIANS_PER_DEGREE, Vectors::UNIT_Y) * + glm::angleAxis(euler.x * RADIANS_PER_DEGREE, Vectors::UNIT_X) * + glm::angleAxis(euler.z * RADIANS_PER_DEGREE, Vectors::UNIT_Z)); +} + +void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { + + std::vector joints; + // copy the shared data + { + std::lock_guard guard(_jointsMutex); + joints = _joints; + } + + DebugDraw::getInstance().addMyAvatarMarker("LEFT_FOOT", + eulerToQuat(joints[6].rot), + joints[6].pos / 100.0f, + glm::vec4(1)); + + _inputDevice->update(deltaTime, jointsCaptured); +} + +void NeuronPlugin::saveSettings() const { + InputPlugin::saveSettings(); + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::saveSettings"; +} + +void NeuronPlugin::loadSettings() { + InputPlugin::loadSettings(); + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::loadSettings"; +} + +// +// InputDevice +// + +controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const { + // TODO: + static const controller::Input::NamedVector availableInputs { + makePair(controller::LEFT_HAND, "LeftHand"), + makePair(controller::RIGHT_HAND, "RightHand") + }; + return availableInputs; +} + +QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { + static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/neuron.json"; + return MAPPING_JSON; +} + +void NeuronPlugin::InputDevice::update(float deltaTime, bool jointsCaptured) { + +} + +void NeuronPlugin::InputDevice::focusOutEvent() { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::InputDevice::focusOutEvent"; +} diff --git a/plugins/neuron/src/NeuronPlugin.h b/plugins/hifiNeuron/src/NeuronPlugin.h similarity index 79% rename from plugins/neuron/src/NeuronPlugin.h rename to plugins/hifiNeuron/src/NeuronPlugin.h index 59e0f0a393..5f67502e04 100644 --- a/plugins/neuron/src/NeuronPlugin.h +++ b/plugins/hifiNeuron/src/NeuronPlugin.h @@ -16,10 +16,15 @@ #include #include +struct _BvhDataHeaderEx; +void FrameDataReceivedCallback(void* context, void* sender, _BvhDataHeaderEx* header, float* data); + // Handles interaction with the Neuron SDK class NeuronPlugin : public InputPlugin { Q_OBJECT public: + friend void FrameDataReceivedCallback(void* context, void* sender, _BvhDataHeaderEx* header, float* data); + // Plugin functions virtual bool isSupported() const override; virtual bool isJointController() const override { return true; } @@ -35,7 +40,7 @@ public: virtual void saveSettings() const override; virtual void loadSettings() override; -private: +protected: class InputDevice : public controller::InputDevice { public: InputDevice() : controller::InputDevice("Neuron") {} @@ -51,6 +56,18 @@ private: static const QString NAME; static const QString NEURON_ID_STRING; + + std::string _serverAddress; + int _serverPort; + void* _socketRef; + + struct NeuronJoint { + glm::vec3 pos; + glm::vec3 rot; + }; + + std::vector _joints; + std::mutex _jointsMutex; }; #endif // hifi_NeuronPlugin_h diff --git a/plugins/neuron/src/NeuronProvider.cpp b/plugins/hifiNeuron/src/NeuronProvider.cpp similarity index 100% rename from plugins/neuron/src/NeuronProvider.cpp rename to plugins/hifiNeuron/src/NeuronProvider.cpp diff --git a/plugins/neuron/src/plugin.json b/plugins/hifiNeuron/src/plugin.json similarity index 100% rename from plugins/neuron/src/plugin.json rename to plugins/hifiNeuron/src/plugin.json diff --git a/plugins/neuron/src/NeuronPlugin.cpp b/plugins/neuron/src/NeuronPlugin.cpp deleted file mode 100644 index 735b81a1ef..0000000000 --- a/plugins/neuron/src/NeuronPlugin.cpp +++ /dev/null @@ -1,75 +0,0 @@ -// -// NeuronPlugin.h -// input-plugins/src/input-plugins -// -// Created by Anthony Thibault on 12/18/2015. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "NeuronPlugin.h" - -#include -#include - -Q_DECLARE_LOGGING_CATEGORY(inputplugins) -Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") - -const QString NeuronPlugin::NAME = "Neuron"; -const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; - -bool NeuronPlugin::isSupported() const { - // TODO: - return true; -} - -void NeuronPlugin::activate() { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::activate"; -} - -void NeuronPlugin::deactivate() { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::deactivate"; -} - -void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::pluginUpdate"; -} - -void NeuronPlugin::saveSettings() const { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::saveSettings"; -} - -void NeuronPlugin::loadSettings() { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::loadSettings"; -} - -controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const { - // TODO: - static const controller::Input::NamedVector availableInputs { - makePair(controller::LEFT_HAND, "LeftHand"), - makePair(controller::RIGHT_HAND, "RightHand") - }; - return availableInputs; -} - -QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { - static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/neuron.json"; - return MAPPING_JSON; -} - -void NeuronPlugin::InputDevice::update(float deltaTime, bool jointsCaptured) { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::InputDevice::update"; -} - -void NeuronPlugin::InputDevice::focusOutEvent() { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::InputDevice::focusOutEvent"; -} From 6afe3bae5ecfe0bb7096391a29e8d7ce26326591 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 22 Dec 2015 17:21:33 -0800 Subject: [PATCH 03/13] Copy Neuron joints into controller poses This makes the accessible for controller mapping and to JavaScript. Added 'neuronAvatar.js' as an example of reading joints from the neuron and setting them on the avatar. NOTE: the rotations are currently in the wrong coordinate frame. --- examples/controllers/neuron/neuronAvatar.js | 141 ++++++++++++ libraries/animation/src/Rig.cpp | 6 +- .../src/controllers/StandardControls.h | 61 +++++- plugins/hifiNeuron/src/NeuronPlugin.cpp | 205 ++++++++++++++++-- plugins/hifiNeuron/src/NeuronPlugin.h | 21 +- 5 files changed, 400 insertions(+), 34 deletions(-) create mode 100644 examples/controllers/neuron/neuronAvatar.js diff --git a/examples/controllers/neuron/neuronAvatar.js b/examples/controllers/neuron/neuronAvatar.js new file mode 100644 index 0000000000..a7146e0759 --- /dev/null +++ b/examples/controllers/neuron/neuronAvatar.js @@ -0,0 +1,141 @@ +// maps controller joint names to avatar joint names +var JOINT_NAME_MAP = { + HipsPosition: "", + Hips: "Hips", + RightUpLeg: "RightUpLeg", + RightLeg: "RightLeg", + RightFoot: "RightFoot", + LeftUpLeg: "LeftUpLeg", + LeftLeg: "LeftLeg", + LeftFoot: "LeftFoot", + Spine: "Spine", + Spine1: "Spine1", + Spine2: "Spine2", + Spine3: "Spine3", + Neck: "Neck", + Head: "Head", + RightShoulder: "RightShoulder", + RightArm: "RightArm", + RightForeArm: "RightForeArm", + RightHand: "RightHand", + RightHandThumb1: "RightHandThumb2", + RightHandThumb2: "RightHandThumb3", + RightHandThumb3: "RightHandThumb4", + RightInHandIndex: "RightHandIndex1", + RightHandIndex1: "RightHandIndex2", + RightHandIndex2: "RightHandIndex3", + RightHandIndex3: "RightHandIndex4", + RightInHandMiddle: "RightHandMiddle1", + RightHandMiddle1: "RightHandMiddle2", + RightHandMiddle2: "RightHandMiddle3", + RightHandMiddle3: "RightHandMiddle4", + RightInHandRing: "RightHandRing1", + RightHandRing1: "RightHandRing2", + RightHandRing2: "RightHandRing3", + RightHandRing3: "RightHandRing4", + RightInHandPinky: "RightHandPinky1", + RightHandPinky1: "RightHandPinky2", + RightHandPinky2: "RightHandPinky3", + RightHandPinky3: "RightHandPinky4", + LeftShoulder: "LeftShoulder", + LeftArm: "LeftArm", + LeftForeArm: "LeftForeArm", + LeftHand: "LeftHand", + LeftHandThumb1: "LeftHandThumb2", + LeftHandThumb2: "LeftHandThumb3", + LeftHandThumb3: "LeftHandThumb4", + LeftInHandIndex: "LeftHandIndex1", + LeftHandIndex1: "LeftHandIndex2", + LeftHandIndex2: "LeftHandIndex3", + LeftHandIndex3: "LeftHandIndex4", + LeftInHandMiddle: "LeftHandMiddle1", + LeftHandMiddle1: "LeftHandMiddle2", + LeftHandMiddle2: "LeftHandMiddle3", + LeftHandMiddle3: "LeftHandMiddle4", + LeftInHandRing: "LeftHandRing1", + LeftHandRing1: "LeftHandRing2", + LeftHandRing2: "LeftHandRing3", + LeftHandRing3: "LeftHandRing4", + LeftInHandPinky: "LeftHandPinky1", + LeftHandPinky1: "LeftHandPinky2", + LeftHandPinky2: "LeftHandPinky3", + LeftHandPinky3: "LeftHandPinky4" +}; + +function dumpHardwareMapping() { + Object.keys(Controller.Hardware).forEach(function (deviceName) { + Object.keys(Controller.Hardware[deviceName]).forEach(function (input) { + print("Controller.Hardware." + deviceName + "." + input + ":" + Controller.Hardware[deviceName][input]); + }); + }); +} + +// ctor +function NeuronAvatar() { + var self = this; + Script.scriptEnding.connect(function () { + self.shutdown(); + }); + Controller.hardwareChanged.connect(function () { + self.hardwareChanged(); + }); + + if (Controller.Hardware.Neuron) { + this.activate(); + } else { + this.deactivate(); + } +} + +NeuronAvatar.prototype.shutdown = function () { + this.deactivate(); +}; + +NeuronAvatar.prototype.hardwareChanged = function () { + if (Controller.Hardware.Neuron) { + this.activate(); + } else { + this.deactivate(); + } +}; + +NeuronAvatar.prototype.activate = function () { + if (!this._active) { + Script.update.connect(updateCallback); + } + this._active = true; +}; + +NeuronAvatar.prototype.deactivate = function () { + if (this._active) { + var self = this; + Script.update.disconnect(updateCallback); + } + this._active = false; + MyAvatar.clearJointsData(); +}; + +NeuronAvatar.prototype.update = function (deltaTime) { + var keys = Object.keys(JOINT_NAME_MAP); + var i, l = keys.length; + for (i = 0; i < l; i++) { + var channel = Controller.Hardware.Neuron[keys[i]]; + if (channel) { + var pose = Controller.getPoseValue(channel); + var j = MyAvatar.getJointIndex(JOINT_NAME_MAP[keys[i]]); + var defaultRot = MyAvatar.getDefaultJointRotation(j); + if (keys[i] == "Hips") { + MyAvatar.setJointRotation(j, Quat.multiply(pose.rotation, defaultRot)); + } else { + MyAvatar.setJointRotation(j, defaultRot); + } + } + } +}; + +var neuronAvatar = new NeuronAvatar(); + +function updateCallback(deltaTime) { + neuronAvatar.update(deltaTime); +} + diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index 68f382d2d9..4dd091f1d6 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -289,8 +289,10 @@ void Rig::clearJointState(int index) { void Rig::clearJointStates() { _internalPoseSet._overrideFlags.clear(); - _internalPoseSet._overrideFlags.resize(_animSkeleton->getNumJoints()); - _internalPoseSet._overridePoses = _animSkeleton->getRelativeDefaultPoses(); + if (_animSkeleton) { + _internalPoseSet._overrideFlags.resize(_animSkeleton->getNumJoints()); + _internalPoseSet._overridePoses = _animSkeleton->getRelativeDefaultPoses(); + } } void Rig::clearJointAnimationPriority(int index) { diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index bbd33c5cb3..feed8a0fad 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -88,9 +88,66 @@ namespace controller { // No correlation to SDL enum StandardPoseChannel { - LEFT_HAND = 0, - RIGHT_HAND, + HIPS_ROOT = 0, + HIPS, + RIGHT_UP_LEG, + RIGHT_LEG, + RIGHT_FOOT, + LEFT_UP_LEG, + LEFT_LEG, + LEFT_FOOT, + SPINE, + SPINE1, + SPINE2, + SPINE3, + NECK, HEAD, + RIGHT_SHOULDER, + RIGHT_ARM, + RIGHT_FORE_ARM, + RIGHT_HAND, + RIGHT_HAND_THUMB1, + RIGHT_HAND_THUMB2, + RIGHT_HAND_THUMB3, + RIGHT_IN_HAND_INDEX, + RIGHT_HAND_INDEX1, + RIGHT_HAND_INDEX2, + RIGHT_HAND_INDEX3, + RIGHT_IN_HAND_MIDDLE, + RIGHT_HAND_MIDDLE1, + RIGHT_HAND_MIDDLE2, + RIGHT_HAND_MIDDLE3, + RIGHT_IN_HANDRING, + RIGHT_HAND_RING1, + RIGHT_HAND_RING2, + RIGHT_HAND_RING3, + RIGHT_IN_HAND_PINKY, + RIGHT_HAND_PINKY1, + RIGHT_HAND_PINKY2, + RIGHT_HAND_PINKY3, + LEFT_SHOULDER, + LEFT_ARM, + LEFT_FORE_ARM, + LEFT_HAND, + LEFT_HAND_THUMB1, + LEFT_HAND_THUMB2, + LEFT_HAND_THUMB3, + LEFT_IN_HAND_INDEX, + LEFT_HAND_INDEX1, + LEFT_HAND_INDEX2, + LEFT_HAND_INDEX3, + LEFT_IN_HAND_MIDDLE, + LEFT_HAND_MIDDLE1, + LEFT_HAND_MIDDLE2, + LEFT_HAND_MIDDLE3, + LEFT_IN_HAND_RING, + LEFT_HAND_RING1, + LEFT_HAND_RING2, + LEFT_HAND_RING3, + LEFT_IN_HAND_PINKY, + LEFT_HAND_PINKY1, + LEFT_HAND_PINKY2, + LEFT_HAND_PINKY3, NUM_STANDARD_POSES }; diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp index c3f764da05..4f52d8da98 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.cpp +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -1,5 +1,5 @@ // -// NeuronPlugin.h +// NeuronPlugin.cpp // input-plugins/src/input-plugins // // Created by Anthony Thibault on 12/18/2015. @@ -11,6 +11,7 @@ #include "NeuronPlugin.h" +#include #include #include #include @@ -27,6 +28,7 @@ Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") const QString NeuronPlugin::NAME = "Neuron"; const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; +// This matches controller::StandardPoseChannel enum JointIndex { HipsPosition = 0, Hips, @@ -87,7 +89,8 @@ enum JointIndex { LeftInHandPinky, LeftHandPinky1, LeftHandPinky2, - LeftHandPinky3 + LeftHandPinky3, + Size }; bool NeuronPlugin::isSupported() const { @@ -98,29 +101,81 @@ bool NeuronPlugin::isSupported() const { // NOTE: must be thread-safe void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx* header, float* data) { - qCDebug(inputplugins) << "NeuronPlugin: received frame data, DataCount = " << header->DataCount; auto neuronPlugin = reinterpret_cast(context); - std::lock_guard guard(neuronPlugin->_jointsMutex); - // Data is 6 floats: 3 position values, 3 rotation euler angles (degrees) + // version 1.0 + if (header->DataVersion.Major == 1 && header->DataVersion.Minor == 0) { - // resize vector if necessary - const size_t NUM_FLOATS_PER_JOINT = 6; - const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; - if (neuronPlugin->_joints.size() != NUM_JOINTS) { - neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + std::lock_guard guard(neuronPlugin->_jointsMutex); + + // Data is 6 floats: 3 position values, 3 rotation euler angles (degrees) + + // resize vector if necessary + const size_t NUM_FLOATS_PER_JOINT = 6; + const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; + if (neuronPlugin->_joints.size() != NUM_JOINTS) { + neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + } + + assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); + + // copy the data + memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); + + } else { + static bool ONCE = false; + if (!ONCE) { + qCCritical(inputplugins) << "NeuronPlugin: bad frame version number, expected 1.0"; + ONCE = true; + } } - - assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); - - // copy the data - memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); } // NOTE: must be thread-safe static void CommandDataReceivedCallback(void* context, SOCKET_REF sender, CommandPack* pack, void* data) { + DATA_VER version; + version._VersionMask = pack->DataVersion; + if (version.Major == 1 && version.Minor == 0) { + const char* str = "Unknown"; + switch (pack->CommandId) { + case Cmd_BoneSize: // Id can be used to request bone size from server or register avatar name command. + str = "BoneSize"; + break; + case Cmd_AvatarName: // Id can be used to request avatar name from server or register avatar name command. + str = "AvatarName"; + break; + case Cmd_FaceDirection: // Id used to request face direction from server + str = "FaceDirection"; + break; + case Cmd_DataFrequency: // Id can be used to request data frequency from server or register data frequency command. + str = "DataFrequency"; + break; + case Cmd_BvhInheritanceTxt: // Id can be used to request bvh header txt from server or register bvh header txt command. + str = "BvhInheritanceTxt"; + break; + case Cmd_AvatarCount: // Id can be used to request avatar count from server or register avatar count command. + str = "AvatarCount"; + break; + case Cmd_CombinationMode: // Id can be used to request combination mode from server or register combination mode command. + str = "CombinationMode"; + break; + case Cmd_RegisterEvent: // Id can be used to register event. + str = "RegisterEvent"; + break; + case Cmd_UnRegisterEvent: // Id can be used to unregister event. + str = "UnRegisterEvent"; + break; + } + qCDebug(inputplugins) << "NeuronPlugin: command data received CommandID = " << str; + } else { + static bool ONCE = false; + if (!ONCE) { + qCCritical(inputplugins) << "NeuronPlugin: bad command version number, expected 1.0"; + ONCE = true; + } + } } // NOTE: must be thread-safe @@ -130,6 +185,11 @@ static void SocketStatusChangedCallback(void* context, SOCKET_REF sender, Socket void NeuronPlugin::activate() { InputPlugin::activate(); + + // register with userInputMapper + auto userInputMapper = DependencyManager::get(); + userInputMapper->registerDevice(_inputDevice); + qCDebug(inputplugins) << "NeuronPlugin::activate"; // register c-style callbacks @@ -149,12 +209,18 @@ void NeuronPlugin::activate() { } void NeuronPlugin::deactivate() { - // TODO: qCDebug(inputplugins) << "NeuronPlugin::deactivate"; + // unregister from userInputMapper + if (_inputDevice->_deviceID != controller::Input::INVALID_DEVICE) { + auto userInputMapper = DependencyManager::get(); + userInputMapper->removeDevice(_inputDevice->_deviceID); + } + if (_socketRef) { BRCloseSocket(_socketRef); } + InputPlugin::deactivate(); } @@ -174,12 +240,14 @@ void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { joints = _joints; } + /* DebugDraw::getInstance().addMyAvatarMarker("LEFT_FOOT", - eulerToQuat(joints[6].rot), + eulerToQuat(joints[6].euler), joints[6].pos / 100.0f, glm::vec4(1)); - - _inputDevice->update(deltaTime, jointsCaptured); + */ + _inputDevice->update(deltaTime, joints, _prevJoints); + _prevJoints = joints; } void NeuronPlugin::saveSettings() const { @@ -198,11 +266,86 @@ void NeuronPlugin::loadSettings() { // InputDevice // +static controller::StandardPoseChannel neuronJointIndexToPoseIndex(JointIndex i) { + // Currently they are the same. + // but that won't always be the case... + return (controller::StandardPoseChannel)i; +} + +static const char* neuronJointName(JointIndex i) { + switch (i) { + case HipsPosition: return "HipsPosition"; + case Hips: return "Hips"; + case RightUpLeg: return "RightUpLeg"; + case RightLeg: return "RightLeg"; + case RightFoot: return "RightFoot"; + case LeftUpLeg: return "LeftUpLeg"; + case LeftLeg: return "LeftLeg"; + case LeftFoot: return "LeftFoot"; + case Spine: return "Spine"; + case Spine1: return "Spine1"; + case Spine2: return "Spine2"; + case Spine3: return "Spine3"; + case Neck: return "Neck"; + case Head: return "Head"; + case RightShoulder: return "RightShoulder"; + case RightArm: return "RightArm"; + case RightForeArm: return "RightForeArm"; + case RightHand: return "RightHand"; + case RightHandThumb1: return "RightHandThumb1"; + case RightHandThumb2: return "RightHandThumb2"; + case RightHandThumb3: return "RightHandThumb3"; + case RightInHandIndex: return "RightInHandIndex"; + case RightHandIndex1: return "RightHandIndex1"; + case RightHandIndex2: return "RightHandIndex2"; + case RightHandIndex3: return "RightHandIndex3"; + case RightInHandMiddle: return "RightInHandMiddle"; + case RightHandMiddle1: return "RightHandMiddle1"; + case RightHandMiddle2: return "RightHandMiddle2"; + case RightHandMiddle3: return "RightHandMiddle3"; + case RightInHandRing: return "RightInHandRing"; + case RightHandRing1: return "RightHandRing1"; + case RightHandRing2: return "RightHandRing2"; + case RightHandRing3: return "RightHandRing3"; + case RightInHandPinky: return "RightInHandPinky"; + case RightHandPinky1: return "RightHandPinky1"; + case RightHandPinky2: return "RightHandPinky2"; + case RightHandPinky3: return "RightHandPinky3"; + case LeftShoulder: return "LeftShoulder"; + case LeftArm: return "LeftArm"; + case LeftForeArm: return "LeftForeArm"; + case LeftHand: return "LeftHand"; + case LeftHandThumb1: return "LeftHandThumb1"; + case LeftHandThumb2: return "LeftHandThumb2"; + case LeftHandThumb3: return "LeftHandThumb3"; + case LeftInHandIndex: return "LeftInHandIndex"; + case LeftHandIndex1: return "LeftHandIndex1"; + case LeftHandIndex2: return "LeftHandIndex2"; + case LeftHandIndex3: return "LeftHandIndex3"; + case LeftInHandMiddle: return "LeftInHandMiddle"; + case LeftHandMiddle1: return "LeftHandMiddle1"; + case LeftHandMiddle2: return "LeftHandMiddle2"; + case LeftHandMiddle3: return "LeftHandMiddle3"; + case LeftInHandRing: return "LeftInHandRing"; + case LeftHandRing1: return "LeftHandRing1"; + case LeftHandRing2: return "LeftHandRing2"; + case LeftHandRing3: return "LeftHandRing3"; + case LeftInHandPinky: return "LeftInHandPinky"; + case LeftHandPinky1: return "LeftHandPinky1"; + case LeftHandPinky2: return "LeftHandPinky2"; + case LeftHandPinky3: return "LeftHandPinky3"; + default: return "???"; + } +} + controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const { // TODO: - static const controller::Input::NamedVector availableInputs { - makePair(controller::LEFT_HAND, "LeftHand"), - makePair(controller::RIGHT_HAND, "RightHand") + static controller::Input::NamedVector availableInputs; + + if (availableInputs.size() == 0) { + for (int i = 0; i < JointIndex::Size; i++) { + availableInputs.push_back(makePair(neuronJointIndexToPoseIndex((JointIndex)i), neuronJointName((JointIndex)i))); + } }; return availableInputs; } @@ -212,8 +355,24 @@ QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { return MAPPING_JSON; } -void NeuronPlugin::InputDevice::update(float deltaTime, bool jointsCaptured) { +void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector& joints, const std::vector& prevJoints) { + for (int i = 0; i < joints.size(); i++) { + int poseIndex = neuronJointIndexToPoseIndex((JointIndex)i); + glm::vec3 linearVel, angularVel; + glm::vec3 pos = (joints[i].pos * METERS_PER_CENTIMETER); + glm::quat rot = eulerToQuat(joints[i].euler); + if (i < prevJoints.size()) { + linearVel = (pos - (prevJoints[i].pos * METERS_PER_CENTIMETER)) / deltaTime; + // quat log imag part points along the axis of rotation, and it's length will be the half angle. + glm::quat d = glm::log(rot * glm::inverse(eulerToQuat(prevJoints[i].euler))); + angularVel = glm::vec3(d.x, d.y, d.z) / (0.5f * deltaTime); + } + _poseStateMap[poseIndex] = controller::Pose(pos, rot, linearVel, angularVel); + if (glm::length(angularVel) > 0.5f) { + qCDebug(inputplugins) << "Movement in joint" << i << neuronJointName((JointIndex)i); + } + } } void NeuronPlugin::InputDevice::focusOutEvent() { diff --git a/plugins/hifiNeuron/src/NeuronPlugin.h b/plugins/hifiNeuron/src/NeuronPlugin.h index 5f67502e04..f787838ce2 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.h +++ b/plugins/hifiNeuron/src/NeuronPlugin.h @@ -41,15 +41,25 @@ public: virtual void loadSettings() override; protected: + + struct NeuronJoint { + glm::vec3 pos; + glm::vec3 euler; + }; + class InputDevice : public controller::InputDevice { public: + friend class NeuronPlugin; + InputDevice() : controller::InputDevice("Neuron") {} // Device functions virtual controller::Input::NamedVector getAvailableInputs() const override; virtual QString getDefaultMappingConfig() const override; - virtual void update(float deltaTime, bool jointsCaptured) override; + virtual void update(float deltaTime, bool jointsCaptured) override {}; virtual void focusOutEvent() override; + + void update(float deltaTime, const std::vector& joints, const std::vector& prevJoints); }; std::shared_ptr _inputDevice { std::make_shared() }; @@ -61,13 +71,10 @@ protected: int _serverPort; void* _socketRef; - struct NeuronJoint { - glm::vec3 pos; - glm::vec3 rot; - }; - std::vector _joints; - std::mutex _jointsMutex; + std::mutex _jointsMutex; // used to guard access to _joints + + std::vector _prevJoints; }; #endif // hifi_NeuronPlugin_h From c44f69b370387b06ab16b27583aaf5940f3cf59d Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 23 Dec 2015 17:13:52 -0800 Subject: [PATCH 04/13] WIP checkpoint Changed euler angle compisition based on experiments in maya. Also, the neuronAvatar.js attempts to transform the neuron input quaternions into a pose relative to the avatar's default pose, but doesn't it doesn't work. --- examples/controllers/neuron/neuronAvatar.js | 9 +++----- .../src/controllers/StandardControls.h | 3 +-- plugins/hifiNeuron/src/NeuronPlugin.cpp | 22 +++++++++++++------ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/examples/controllers/neuron/neuronAvatar.js b/examples/controllers/neuron/neuronAvatar.js index a7146e0759..776cabd301 100644 --- a/examples/controllers/neuron/neuronAvatar.js +++ b/examples/controllers/neuron/neuronAvatar.js @@ -1,6 +1,5 @@ // maps controller joint names to avatar joint names var JOINT_NAME_MAP = { - HipsPosition: "", Hips: "Hips", RightUpLeg: "RightUpLeg", RightLeg: "RightLeg", @@ -124,11 +123,9 @@ NeuronAvatar.prototype.update = function (deltaTime) { var pose = Controller.getPoseValue(channel); var j = MyAvatar.getJointIndex(JOINT_NAME_MAP[keys[i]]); var defaultRot = MyAvatar.getDefaultJointRotation(j); - if (keys[i] == "Hips") { - MyAvatar.setJointRotation(j, Quat.multiply(pose.rotation, defaultRot)); - } else { - MyAvatar.setJointRotation(j, defaultRot); - } + var rot = Quat.multiply(Quat.inverse(defaultRot), Quat.multiply(pose.rotation, defaultRot)); + MyAvatar.setJointRotation(j, Quat.multiply(defaultRot, rot)); + //MyAvatar.setJointTranslation(j, Vec3.multiply(100.0, pose.translation)); } } }; diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index feed8a0fad..4294713238 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -88,8 +88,7 @@ namespace controller { // No correlation to SDL enum StandardPoseChannel { - HIPS_ROOT = 0, - HIPS, + HIPS = 0, RIGHT_UP_LEG, RIGHT_LEG, RIGHT_FOOT, diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp index 4f52d8da98..152bce913d 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.cpp +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -17,6 +17,7 @@ #include #include #include +#include Q_DECLARE_LOGGING_CATEGORY(inputplugins) Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") @@ -30,8 +31,7 @@ const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; // This matches controller::StandardPoseChannel enum JointIndex { - HipsPosition = 0, - Hips, + Hips = 0, RightUpLeg, RightLeg, RightFoot, @@ -204,8 +204,9 @@ void NeuronPlugin::activate() { if (!_socketRef) { // error qCCritical(inputplugins) << "NeuronPlugin: error connecting to " << _serverAddress.c_str() << ":" << _serverPort << "error = " << BRGetLastErrorMessage(); + } else { + qCDebug(inputplugins) << "NeuronPlugin: success connecting to " << _serverAddress.c_str() << ":" << _serverPort; } - qCDebug(inputplugins) << "NeuronPlugin: success connecting to " << _serverAddress.c_str() << ":" << _serverPort; } void NeuronPlugin::deactivate() { @@ -226,9 +227,9 @@ void NeuronPlugin::deactivate() { // convert between euler in degrees to quaternion static quat eulerToQuat(vec3 euler) { - return (glm::angleAxis(euler.y * RADIANS_PER_DEGREE, Vectors::UNIT_Y) * - glm::angleAxis(euler.x * RADIANS_PER_DEGREE, Vectors::UNIT_X) * - glm::angleAxis(euler.z * RADIANS_PER_DEGREE, Vectors::UNIT_Z)); + // euler.x and euler.y are swaped (thanks NOMICOM!) + glm::vec3 e = glm::vec3(euler.y, euler.x, euler.z) * RADIANS_PER_DEGREE; + return (glm::angleAxis(e.y, Vectors::UNIT_Y) * glm::angleAxis(e.x, Vectors::UNIT_X) * glm::angleAxis(e.z, Vectors::UNIT_Z)); } void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { @@ -274,7 +275,6 @@ static controller::StandardPoseChannel neuronJointIndexToPoseIndex(JointIndex i) static const char* neuronJointName(JointIndex i) { switch (i) { - case HipsPosition: return "HipsPosition"; case Hips: return "Hips"; case RightUpLeg: return "RightUpLeg"; case RightLeg: return "RightLeg"; @@ -369,9 +369,17 @@ void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector Date: Thu, 24 Dec 2015 17:14:17 -0800 Subject: [PATCH 05/13] neruonAvatar.js: now sets rotations in correct frame. The rotations from the neuron are effectively in world space and are deltas from the default pose. There still is an issue with the thumb, due to the missing joint from the Neuron. The Neuron only has 3 thumb joints, not 4. --- examples/controllers/neuron/neuronAvatar.js | 95 ++++++++++++++++++++- plugins/hifiNeuron/src/NeuronPlugin.cpp | 52 +++++------ 2 files changed, 119 insertions(+), 28 deletions(-) diff --git a/examples/controllers/neuron/neuronAvatar.js b/examples/controllers/neuron/neuronAvatar.js index 776cabd301..fae46330b1 100644 --- a/examples/controllers/neuron/neuronAvatar.js +++ b/examples/controllers/neuron/neuronAvatar.js @@ -61,6 +61,70 @@ var JOINT_NAME_MAP = { LeftHandPinky3: "LeftHandPinky4" }; +var JOINT_PARENT_MAP = { + Hips: "", + RightUpLeg: "Hips", + RightLeg: "RightUpLeg", + RightFoot: "RightLeg", + LeftUpLeg: "Hips", + LeftLeg: "LeftUpLeg", + LeftFoot: "LeftLeg", + Spine: "Hips", + Spine1: "Spine", + Spine2: "Spine1", + Spine3: "Spine2", + Neck: "Spine3", + Head: "Neck", + RightShoulder: "Spine3", + RightArm: "RightShoulder", + RightForeArm: "RightArm", + RightHand: "RightForeArm", + RightHandThumb1: "RightHand", + RightHandThumb2: "RightHandThumb1", + RightHandThumb3: "RightHandThumb2", + RightHandThumb4: "RightHandThumb3", + RightHandIndex1: "RightHand", + RightHandIndex2: "RightHandIndex1", + RightHandIndex3: "RightHandIndex2", + RightHandIndex4: "RightHandIndex3", + RightHandMiddle1: "RightHand", + RightHandMiddle2: "RightHandMiddle1", + RightHandMiddle3: "RightHandMiddle2", + RightHandMiddle4: "RightHandMiddle3", + RightHandRing1: "RightHand", + RightHandRing2: "RightHandRing1", + RightHandRing3: "RightHandRing2", + RightHandRing4: "RightHandRing3", + RightHandPinky1: "RightHand", + RightHandPinky2: "RightHandPinky1", + RightHandPinky3: "RightHandPinky2", + RightHandPinky4: "RightHandPinky3", + LeftShoulder: "Spine3", + LeftArm: "LeftShoulder", + LeftForeArm: "LeftArm", + LeftHand: "LeftForeArm", + LeftHandThumb1: "LeftHand", + LeftHandThumb2: "LeftHandThumb1", + LeftHandThumb3: "LeftHandThumb2", + LeftHandThumb4: "LeftHandThumb3", + LeftHandIndex1: "LeftHand", + LeftHandIndex2: "LeftHandIndex1", + LeftHandIndex3: "LeftHandIndex2", + LeftHandIndex4: "LeftHandIndex3", + LeftHandMiddle1: "LeftHand", + LeftHandMiddle2: "LeftHandMiddle1", + LeftHandMiddle3: "LeftHandMiddle2", + LeftHandMiddle4: "LeftHandMiddle3", + LeftHandRing1: "LeftHand", + LeftHandRing2: "LeftHandRing1", + LeftHandRing3: "LeftHandRing2", + LeftHandRing4: "LeftHandRing3", + LeftHandPinky1: "LeftHand", + LeftHandPinky2: "LeftHandPinky1", + LeftHandPinky3: "LeftHandPinky2", + LeftHandPinky: "LeftHandPinky3", +}; + function dumpHardwareMapping() { Object.keys(Controller.Hardware).forEach(function (deviceName) { Object.keys(Controller.Hardware[deviceName]).forEach(function (input) { @@ -103,6 +167,21 @@ NeuronAvatar.prototype.activate = function () { Script.update.connect(updateCallback); } this._active = true; + + // build absDefaultPoseMap + this._absDefaultRotMap = {}; + var keys = Object.keys(JOINT_NAME_MAP); + var i, l = keys.length; + for (i = 0; i < l; i++) { + var jointName = JOINT_NAME_MAP[keys[i]]; + var j = MyAvatar.getJointIndex(jointName); + var parentJointName = JOINT_PARENT_MAP[jointName]; + if (parentJointName === "") { + this._absDefaultRotMap[jointName] = MyAvatar.getDefaultJointRotation(j); + } else { + this._absDefaultRotMap[jointName] = Quat.multiply(this._absDefaultRotMap[parentJointName], MyAvatar.getDefaultJointRotation(j)); + } + } }; NeuronAvatar.prototype.deactivate = function () { @@ -117,14 +196,22 @@ NeuronAvatar.prototype.deactivate = function () { NeuronAvatar.prototype.update = function (deltaTime) { var keys = Object.keys(JOINT_NAME_MAP); var i, l = keys.length; + var absDefaultRot = {}; for (i = 0; i < l; i++) { var channel = Controller.Hardware.Neuron[keys[i]]; if (channel) { var pose = Controller.getPoseValue(channel); - var j = MyAvatar.getJointIndex(JOINT_NAME_MAP[keys[i]]); - var defaultRot = MyAvatar.getDefaultJointRotation(j); - var rot = Quat.multiply(Quat.inverse(defaultRot), Quat.multiply(pose.rotation, defaultRot)); - MyAvatar.setJointRotation(j, Quat.multiply(defaultRot, rot)); + var jointName = JOINT_NAME_MAP[keys[i]]; + var parentJointName = JOINT_PARENT_MAP[jointName]; + var j = MyAvatar.getJointIndex(jointName); + var defaultAbsRot = this._absDefaultRotMap[jointName]; + var parentDefaultAbsRot; + if (parentJointName === "") { + parentDefaultAbsRot = {x: 0, y: 0, z: 0, w: 1}; + } else { + parentDefaultAbsRot = this._absDefaultRotMap[parentJointName]; + } + MyAvatar.setJointRotation(j, Quat.multiply(Quat.inverse(parentDefaultAbsRot), Quat.multiply(pose.rotation, defaultAbsRot))); //MyAvatar.setJointTranslation(j, Vec3.multiply(100.0, pose.translation)); } } diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp index 152bce913d..ae438c10e0 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.cpp +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -107,21 +107,37 @@ void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx // version 1.0 if (header->DataVersion.Major == 1 && header->DataVersion.Minor == 0) { - std::lock_guard guard(neuronPlugin->_jointsMutex); - - // Data is 6 floats: 3 position values, 3 rotation euler angles (degrees) - - // resize vector if necessary - const size_t NUM_FLOATS_PER_JOINT = 6; - const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; - if (neuronPlugin->_joints.size() != NUM_JOINTS) { - neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + // skip reference joint if present + if (header->WithReference && header->WithDisp) { + data += 6; + } else if (header->WithReference && !header->WithDisp) { + data += 3; } - assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); + if (header->WithDisp) { + // enter mutex + std::lock_guard guard(neuronPlugin->_jointsMutex); - // copy the data - memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); + // + // Data is 6 floats per joint: 3 position values, 3 rotation euler angles (degrees) + // + + // resize vector if necessary + const size_t NUM_FLOATS_PER_JOINT = 6; + const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; + if (neuronPlugin->_joints.size() != NUM_JOINTS) { + neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + } + + assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); + + // copy the data + memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); + } else { + + qCCritical(inputplugins) << "NeruonPlugin: format not supported"; + + } } else { static bool ONCE = false; @@ -368,18 +384,6 @@ void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector Date: Fri, 25 Dec 2015 09:57:50 -0800 Subject: [PATCH 06/13] Set up controller poses to match hifi standard skeleton Neuron plugin in fills in the gap (thumb1) with identity. Updated neuronAvatar script to work with new controller pose names. --- examples/controllers/neuron/neuronAvatar.js | 97 +----- .../src/controllers/StandardControls.h | 20 +- plugins/hifiNeuron/src/NeuronPlugin.cpp | 324 +++++++++++------- plugins/hifiNeuron/src/NeuronPlugin.h | 10 +- 4 files changed, 245 insertions(+), 206 deletions(-) diff --git a/examples/controllers/neuron/neuronAvatar.js b/examples/controllers/neuron/neuronAvatar.js index fae46330b1..cc53987f05 100644 --- a/examples/controllers/neuron/neuronAvatar.js +++ b/examples/controllers/neuron/neuronAvatar.js @@ -1,66 +1,3 @@ -// maps controller joint names to avatar joint names -var JOINT_NAME_MAP = { - Hips: "Hips", - RightUpLeg: "RightUpLeg", - RightLeg: "RightLeg", - RightFoot: "RightFoot", - LeftUpLeg: "LeftUpLeg", - LeftLeg: "LeftLeg", - LeftFoot: "LeftFoot", - Spine: "Spine", - Spine1: "Spine1", - Spine2: "Spine2", - Spine3: "Spine3", - Neck: "Neck", - Head: "Head", - RightShoulder: "RightShoulder", - RightArm: "RightArm", - RightForeArm: "RightForeArm", - RightHand: "RightHand", - RightHandThumb1: "RightHandThumb2", - RightHandThumb2: "RightHandThumb3", - RightHandThumb3: "RightHandThumb4", - RightInHandIndex: "RightHandIndex1", - RightHandIndex1: "RightHandIndex2", - RightHandIndex2: "RightHandIndex3", - RightHandIndex3: "RightHandIndex4", - RightInHandMiddle: "RightHandMiddle1", - RightHandMiddle1: "RightHandMiddle2", - RightHandMiddle2: "RightHandMiddle3", - RightHandMiddle3: "RightHandMiddle4", - RightInHandRing: "RightHandRing1", - RightHandRing1: "RightHandRing2", - RightHandRing2: "RightHandRing3", - RightHandRing3: "RightHandRing4", - RightInHandPinky: "RightHandPinky1", - RightHandPinky1: "RightHandPinky2", - RightHandPinky2: "RightHandPinky3", - RightHandPinky3: "RightHandPinky4", - LeftShoulder: "LeftShoulder", - LeftArm: "LeftArm", - LeftForeArm: "LeftForeArm", - LeftHand: "LeftHand", - LeftHandThumb1: "LeftHandThumb2", - LeftHandThumb2: "LeftHandThumb3", - LeftHandThumb3: "LeftHandThumb4", - LeftInHandIndex: "LeftHandIndex1", - LeftHandIndex1: "LeftHandIndex2", - LeftHandIndex2: "LeftHandIndex3", - LeftHandIndex3: "LeftHandIndex4", - LeftInHandMiddle: "LeftHandMiddle1", - LeftHandMiddle1: "LeftHandMiddle2", - LeftHandMiddle2: "LeftHandMiddle3", - LeftHandMiddle3: "LeftHandMiddle4", - LeftInHandRing: "LeftHandRing1", - LeftHandRing1: "LeftHandRing2", - LeftHandRing2: "LeftHandRing3", - LeftHandRing3: "LeftHandRing4", - LeftInHandPinky: "LeftHandPinky1", - LeftHandPinky1: "LeftHandPinky2", - LeftHandPinky2: "LeftHandPinky3", - LeftHandPinky3: "LeftHandPinky4" -}; - var JOINT_PARENT_MAP = { Hips: "", RightUpLeg: "Hips", @@ -170,10 +107,11 @@ NeuronAvatar.prototype.activate = function () { // build absDefaultPoseMap this._absDefaultRotMap = {}; - var keys = Object.keys(JOINT_NAME_MAP); + this._absDefaultRotMap[""] = {x: 0, y: 0, z: 0, w: 1}; + var keys = Object.keys(JOINT_PARENT_MAP); var i, l = keys.length; for (i = 0; i < l; i++) { - var jointName = JOINT_NAME_MAP[keys[i]]; + var jointName = keys[i]; var j = MyAvatar.getJointIndex(jointName); var parentJointName = JOINT_PARENT_MAP[jointName]; if (parentJointName === "") { @@ -194,24 +132,27 @@ NeuronAvatar.prototype.deactivate = function () { }; NeuronAvatar.prototype.update = function (deltaTime) { - var keys = Object.keys(JOINT_NAME_MAP); + var keys = Object.keys(JOINT_PARENT_MAP); var i, l = keys.length; var absDefaultRot = {}; + var jointName, channel, pose, parentJointName, j, parentDefaultAbsRot; for (i = 0; i < l; i++) { - var channel = Controller.Hardware.Neuron[keys[i]]; + var jointName = keys[i]; + var channel = Controller.Hardware.Neuron[jointName]; if (channel) { - var pose = Controller.getPoseValue(channel); - var jointName = JOINT_NAME_MAP[keys[i]]; - var parentJointName = JOINT_PARENT_MAP[jointName]; - var j = MyAvatar.getJointIndex(jointName); - var defaultAbsRot = this._absDefaultRotMap[jointName]; - var parentDefaultAbsRot; - if (parentJointName === "") { - parentDefaultAbsRot = {x: 0, y: 0, z: 0, w: 1}; - } else { - parentDefaultAbsRot = this._absDefaultRotMap[parentJointName]; - } + pose = Controller.getPoseValue(channel); + parentJointName = JOINT_PARENT_MAP[jointName]; + j = MyAvatar.getJointIndex(jointName); + defaultAbsRot = this._absDefaultRotMap[jointName]; + parentDefaultAbsRot = this._absDefaultRotMap[parentJointName]; + + // Rotations from the neuron controller are in world orientation but are delta's from the default pose. + // So first we build the absolute rotation of the default pose (local into world). + // Then apply the rotation from the controller, in world space. + // Then we transform back into joint local by multiplying by the inverse of the parents absolute rotation. MyAvatar.setJointRotation(j, Quat.multiply(Quat.inverse(parentDefaultAbsRot), Quat.multiply(pose.rotation, defaultAbsRot))); + + // TODO: //MyAvatar.setJointTranslation(j, Vec3.multiply(100.0, pose.translation)); } } diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index 4294713238..2b0613321e 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -108,22 +108,23 @@ namespace controller { RIGHT_HAND_THUMB1, RIGHT_HAND_THUMB2, RIGHT_HAND_THUMB3, - RIGHT_IN_HAND_INDEX, + RIGHT_HAND_THUMB4, RIGHT_HAND_INDEX1, RIGHT_HAND_INDEX2, RIGHT_HAND_INDEX3, - RIGHT_IN_HAND_MIDDLE, + RIGHT_HAND_INDEX4, RIGHT_HAND_MIDDLE1, RIGHT_HAND_MIDDLE2, RIGHT_HAND_MIDDLE3, - RIGHT_IN_HANDRING, + RIGHT_HAND_MIDDLE4, RIGHT_HAND_RING1, RIGHT_HAND_RING2, RIGHT_HAND_RING3, - RIGHT_IN_HAND_PINKY, + RIGHT_HAND_RING4, RIGHT_HAND_PINKY1, RIGHT_HAND_PINKY2, RIGHT_HAND_PINKY3, + RIGHT_HAND_PINKY4, LEFT_SHOULDER, LEFT_ARM, LEFT_FORE_ARM, @@ -131,28 +132,29 @@ namespace controller { LEFT_HAND_THUMB1, LEFT_HAND_THUMB2, LEFT_HAND_THUMB3, - LEFT_IN_HAND_INDEX, + LEFT_HAND_THUMB4, LEFT_HAND_INDEX1, LEFT_HAND_INDEX2, LEFT_HAND_INDEX3, - LEFT_IN_HAND_MIDDLE, + LEFT_HAND_INDEX4, LEFT_HAND_MIDDLE1, LEFT_HAND_MIDDLE2, LEFT_HAND_MIDDLE3, - LEFT_IN_HAND_RING, + LEFT_HAND_MIDDLE4, LEFT_HAND_RING1, LEFT_HAND_RING2, LEFT_HAND_RING3, - LEFT_IN_HAND_PINKY, + LEFT_HAND_RING4, LEFT_HAND_PINKY1, LEFT_HAND_PINKY2, LEFT_HAND_PINKY3, + LEFT_HAND_PINKY4, NUM_STANDARD_POSES }; enum StandardCounts { TRIGGERS = 2, ANALOG_STICKS = 2, - POSES = 2, // FIXME 3? if we want to expose the head? + POSES = NUM_STANDARD_POSES }; } diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp index ae438c10e0..4edda64c22 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.cpp +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -29,8 +29,10 @@ Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") const QString NeuronPlugin::NAME = "Neuron"; const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; -// This matches controller::StandardPoseChannel -enum JointIndex { +// indices of joints of the Neuron standard skeleton. +// This is 'almost' the same as the High Fidelity standard skeleton. +// It is missing a thumb joint. +enum NeuronJointIndex { Hips = 0, RightUpLeg, RightLeg, @@ -93,12 +95,160 @@ enum JointIndex { Size }; -bool NeuronPlugin::isSupported() const { - // Because it's a client/server network architecture, we can't tell - // if the neuron is actually connected until we connect to the server. - return true; +// Almost a direct mapping except for LEFT_HAND_THUMB1 and RIGHT_HAND_THUMB1, +// which are not present in the Neuron standard skeleton. +static controller::StandardPoseChannel neuronJointIndexToPoseIndexMap[NeuronJointIndex::Size] = { + controller::HIPS, + controller::RIGHT_UP_LEG, + controller::RIGHT_LEG, + controller::RIGHT_FOOT, + controller::LEFT_UP_LEG, + controller::LEFT_LEG, + controller::LEFT_FOOT, + controller::SPINE, + controller::SPINE1, + controller::SPINE2, + controller::SPINE3, + controller::NECK, + controller::HEAD, + controller::RIGHT_SHOULDER, + controller::RIGHT_ARM, + controller::RIGHT_FORE_ARM, + controller::RIGHT_HAND, + controller::RIGHT_HAND_THUMB2, + controller::RIGHT_HAND_THUMB3, + controller::RIGHT_HAND_THUMB4, + controller::RIGHT_HAND_INDEX1, + controller::RIGHT_HAND_INDEX2, + controller::RIGHT_HAND_INDEX3, + controller::RIGHT_HAND_INDEX4, + controller::RIGHT_HAND_MIDDLE1, + controller::RIGHT_HAND_MIDDLE2, + controller::RIGHT_HAND_MIDDLE3, + controller::RIGHT_HAND_MIDDLE4, + controller::RIGHT_HAND_RING1, + controller::RIGHT_HAND_RING2, + controller::RIGHT_HAND_RING3, + controller::RIGHT_HAND_RING4, + controller::RIGHT_HAND_PINKY1, + controller::RIGHT_HAND_PINKY2, + controller::RIGHT_HAND_PINKY3, + controller::RIGHT_HAND_PINKY4, + controller::LEFT_SHOULDER, + controller::LEFT_ARM, + controller::LEFT_FORE_ARM, + controller::LEFT_HAND, + controller::LEFT_HAND_THUMB2, + controller::LEFT_HAND_THUMB3, + controller::LEFT_HAND_THUMB4, + controller::LEFT_HAND_INDEX1, + controller::LEFT_HAND_INDEX2, + controller::LEFT_HAND_INDEX3, + controller::LEFT_HAND_INDEX4, + controller::LEFT_HAND_MIDDLE1, + controller::LEFT_HAND_MIDDLE2, + controller::LEFT_HAND_MIDDLE3, + controller::LEFT_HAND_MIDDLE4, + controller::LEFT_HAND_RING1, + controller::LEFT_HAND_RING2, + controller::LEFT_HAND_RING3, + controller::LEFT_HAND_RING4, + controller::LEFT_HAND_PINKY1, + controller::LEFT_HAND_PINKY2, + controller::LEFT_HAND_PINKY3, + controller::LEFT_HAND_PINKY4 +}; + +static controller::StandardPoseChannel neuronJointIndexToPoseIndex(NeuronJointIndex i) { + assert(i >= 0 && i < NeuronJointIndex::Size); + if (i >= 0 && i < NeuronJointIndex::Size) { + return neuronJointIndexToPoseIndexMap[i]; + } else { + return (controller::StandardPoseChannel)0; // not sure what to do here, but don't crash! + } } +static const char* controllerJointName(controller::StandardPoseChannel i) { + switch (i) { + case controller::HIPS: return "Hips"; + case controller::RIGHT_UP_LEG: return "RightUpLeg"; + case controller::RIGHT_LEG: return "RightLeg"; + case controller::RIGHT_FOOT: return "RightFoot"; + case controller::LEFT_UP_LEG: return "LeftUpLeg"; + case controller::LEFT_LEG: return "LeftLeg"; + case controller::LEFT_FOOT: return "LeftFoot"; + case controller::SPINE: return "Spine"; + case controller::SPINE1: return "Spine1"; + case controller::SPINE2: return "Spine2"; + case controller::SPINE3: return "Spine3"; + case controller::NECK: return "Neck"; + case controller::HEAD: return "Head"; + case controller::RIGHT_SHOULDER: return "RightShoulder"; + case controller::RIGHT_ARM: return "RightArm"; + case controller::RIGHT_FORE_ARM: return "RightForeArm"; + case controller::RIGHT_HAND: return "RightHand"; + case controller::RIGHT_HAND_THUMB1: return "RightHandThumb1"; + case controller::RIGHT_HAND_THUMB2: return "RightHandThumb2"; + case controller::RIGHT_HAND_THUMB3: return "RightHandThumb3"; + case controller::RIGHT_HAND_THUMB4: return "RightHandThumb4"; + case controller::RIGHT_HAND_INDEX1: return "RightHandIndex1"; + case controller::RIGHT_HAND_INDEX2: return "RightHandIndex2"; + case controller::RIGHT_HAND_INDEX3: return "RightHandIndex3"; + case controller::RIGHT_HAND_INDEX4: return "RightHandIndex4"; + case controller::RIGHT_HAND_MIDDLE1: return "RightHandMiddle1"; + case controller::RIGHT_HAND_MIDDLE2: return "RightHandMiddle2"; + case controller::RIGHT_HAND_MIDDLE3: return "RightHandMiddle3"; + case controller::RIGHT_HAND_MIDDLE4: return "RightHandMiddle4"; + case controller::RIGHT_HAND_RING1: return "RightHandRing1"; + case controller::RIGHT_HAND_RING2: return "RightHandRing2"; + case controller::RIGHT_HAND_RING3: return "RightHandRing3"; + case controller::RIGHT_HAND_RING4: return "RightHandRing4"; + case controller::RIGHT_HAND_PINKY1: return "RightHandPinky1"; + case controller::RIGHT_HAND_PINKY2: return "RightHandPinky2"; + case controller::RIGHT_HAND_PINKY3: return "RightHandPinky3"; + case controller::RIGHT_HAND_PINKY4: return "RightHandPinky4"; + case controller::LEFT_SHOULDER: return "LeftShoulder"; + case controller::LEFT_ARM: return "LeftArm"; + case controller::LEFT_FORE_ARM: return "LeftForeArm"; + case controller::LEFT_HAND: return "LeftHand"; + case controller::LEFT_HAND_THUMB1: return "LeftHandThumb1"; + case controller::LEFT_HAND_THUMB2: return "LeftHandThumb2"; + case controller::LEFT_HAND_THUMB3: return "LeftHandThumb3"; + case controller::LEFT_HAND_THUMB4: return "LeftHandThumb4"; + case controller::LEFT_HAND_INDEX1: return "LeftHandIndex1"; + case controller::LEFT_HAND_INDEX2: return "LeftHandIndex2"; + case controller::LEFT_HAND_INDEX3: return "LeftHandIndex3"; + case controller::LEFT_HAND_INDEX4: return "LeftHandIndex4"; + case controller::LEFT_HAND_MIDDLE1: return "LeftHandMiddle1"; + case controller::LEFT_HAND_MIDDLE2: return "LeftHandMiddle2"; + case controller::LEFT_HAND_MIDDLE3: return "LeftHandMiddle3"; + case controller::LEFT_HAND_MIDDLE4: return "LeftHandMiddle4"; + case controller::LEFT_HAND_RING1: return "LeftHandRing1"; + case controller::LEFT_HAND_RING2: return "LeftHandRing2"; + case controller::LEFT_HAND_RING3: return "LeftHandRing3"; + case controller::LEFT_HAND_RING4: return "LeftHandRing4"; + case controller::LEFT_HAND_PINKY1: return "LeftHandPinky1"; + case controller::LEFT_HAND_PINKY2: return "LeftHandPinky2"; + case controller::LEFT_HAND_PINKY3: return "LeftHandPinky3"; + case controller::LEFT_HAND_PINKY4: return "LeftHandPinky4"; + default: return "???"; + } +} + +// convert between YXZ neuron euler angles in degrees to quaternion +// this is the default setting in the Axis Neuron server. +static quat eulerToQuat(vec3 euler) { + // euler.x and euler.y are swaped, WTF. + glm::vec3 e = glm::vec3(euler.y, euler.x, euler.z) * RADIANS_PER_DEGREE; + return (glm::angleAxis(e.y, Vectors::UNIT_Y) * + glm::angleAxis(e.x, Vectors::UNIT_X) * + glm::angleAxis(e.z, Vectors::UNIT_Z)); +} + +// +// neuronDataReader SDK callback functions +// + // NOTE: must be thread-safe void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx* header, float* data) { @@ -133,7 +283,28 @@ void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx // copy the data memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); + } else { + // enter mutex + std::lock_guard guard(neuronPlugin->_jointsMutex); + + // + // Data is 3 floats per joint: 3 rotation euler angles (degrees) + // + + // resize vector if necessary + const size_t NUM_FLOATS_PER_JOINT = 3; + const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; + if (neuronPlugin->_joints.size() != NUM_JOINTS) { + neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + } + + assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); + + for (int i = 0; i < NUM_JOINTS; i++) { + // TODO: should probably just copy over default positions?! + memcpy(&(neuronPlugin->_joints[i].euler), data, sizeof(float) * NUM_FLOATS_PER_JOINT); + } qCCritical(inputplugins) << "NeruonPlugin: format not supported"; @@ -148,6 +319,9 @@ void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx } } +// I can't even get the SDK to send me a callback. +// BRCommandFetchAvatarDataFromServer & BRRegisterAutoSyncParmeter [sic] don't seem to work. +// So this is totally untested. // NOTE: must be thread-safe static void CommandDataReceivedCallback(void* context, SOCKET_REF sender, CommandPack* pack, void* data) { @@ -196,9 +370,20 @@ static void CommandDataReceivedCallback(void* context, SOCKET_REF sender, Comman // NOTE: must be thread-safe static void SocketStatusChangedCallback(void* context, SOCKET_REF sender, SocketStatus status, char* message) { + // just dump to log, later we might want to pop up a connection lost dialog or attempt to reconnect. qCDebug(inputplugins) << "NeuronPlugin: socket status = " << message; } +// +// NeuronPlugin +// + +bool NeuronPlugin::isSupported() const { + // Because it's a client/server network architecture, we can't tell + // if the neuron is actually connected until we connect to the server. + return true; +} + void NeuronPlugin::activate() { InputPlugin::activate(); @@ -206,28 +391,26 @@ void NeuronPlugin::activate() { auto userInputMapper = DependencyManager::get(); userInputMapper->registerDevice(_inputDevice); - qCDebug(inputplugins) << "NeuronPlugin::activate"; - // register c-style callbacks BRRegisterFrameDataCallback((void*)this, FrameDataReceivedCallback); BRRegisterCommandDataCallback((void*)this, CommandDataReceivedCallback); BRRegisterSocketStatusCallback((void*)this, SocketStatusChangedCallback); - // TODO: pull these from prefs! + // TODO: Pull these from prefs dialog? + // localhost is fine for now. _serverAddress = "localhost"; - _serverPort = 7001; + _serverPort = 7001; // default port for TCP Axis Neuron server. + _socketRef = BRConnectTo((char*)_serverAddress.c_str(), _serverPort); if (!_socketRef) { // error - qCCritical(inputplugins) << "NeuronPlugin: error connecting to " << _serverAddress.c_str() << ":" << _serverPort << "error = " << BRGetLastErrorMessage(); + qCCritical(inputplugins) << "NeuronPlugin: error connecting to " << _serverAddress.c_str() << ":" << _serverPort << ", error = " << BRGetLastErrorMessage(); } else { qCDebug(inputplugins) << "NeuronPlugin: success connecting to " << _serverAddress.c_str() << ":" << _serverPort; } } void NeuronPlugin::deactivate() { - qCDebug(inputplugins) << "NeuronPlugin::deactivate"; - // unregister from userInputMapper if (_inputDevice->_deviceID != controller::Input::INVALID_DEVICE) { auto userInputMapper = DependencyManager::get(); @@ -241,126 +424,35 @@ void NeuronPlugin::deactivate() { InputPlugin::deactivate(); } -// convert between euler in degrees to quaternion -static quat eulerToQuat(vec3 euler) { - // euler.x and euler.y are swaped (thanks NOMICOM!) - glm::vec3 e = glm::vec3(euler.y, euler.x, euler.z) * RADIANS_PER_DEGREE; - return (glm::angleAxis(e.y, Vectors::UNIT_Y) * glm::angleAxis(e.x, Vectors::UNIT_X) * glm::angleAxis(e.z, Vectors::UNIT_Z)); -} - void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { - std::vector joints; - // copy the shared data { + // lock and copy std::lock_guard guard(_jointsMutex); joints = _joints; } - - /* - DebugDraw::getInstance().addMyAvatarMarker("LEFT_FOOT", - eulerToQuat(joints[6].euler), - joints[6].pos / 100.0f, - glm::vec4(1)); - */ _inputDevice->update(deltaTime, joints, _prevJoints); _prevJoints = joints; } void NeuronPlugin::saveSettings() const { InputPlugin::saveSettings(); - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::saveSettings"; } void NeuronPlugin::loadSettings() { InputPlugin::loadSettings(); - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::loadSettings"; } // // InputDevice // -static controller::StandardPoseChannel neuronJointIndexToPoseIndex(JointIndex i) { - // Currently they are the same. - // but that won't always be the case... - return (controller::StandardPoseChannel)i; -} - -static const char* neuronJointName(JointIndex i) { - switch (i) { - case Hips: return "Hips"; - case RightUpLeg: return "RightUpLeg"; - case RightLeg: return "RightLeg"; - case RightFoot: return "RightFoot"; - case LeftUpLeg: return "LeftUpLeg"; - case LeftLeg: return "LeftLeg"; - case LeftFoot: return "LeftFoot"; - case Spine: return "Spine"; - case Spine1: return "Spine1"; - case Spine2: return "Spine2"; - case Spine3: return "Spine3"; - case Neck: return "Neck"; - case Head: return "Head"; - case RightShoulder: return "RightShoulder"; - case RightArm: return "RightArm"; - case RightForeArm: return "RightForeArm"; - case RightHand: return "RightHand"; - case RightHandThumb1: return "RightHandThumb1"; - case RightHandThumb2: return "RightHandThumb2"; - case RightHandThumb3: return "RightHandThumb3"; - case RightInHandIndex: return "RightInHandIndex"; - case RightHandIndex1: return "RightHandIndex1"; - case RightHandIndex2: return "RightHandIndex2"; - case RightHandIndex3: return "RightHandIndex3"; - case RightInHandMiddle: return "RightInHandMiddle"; - case RightHandMiddle1: return "RightHandMiddle1"; - case RightHandMiddle2: return "RightHandMiddle2"; - case RightHandMiddle3: return "RightHandMiddle3"; - case RightInHandRing: return "RightInHandRing"; - case RightHandRing1: return "RightHandRing1"; - case RightHandRing2: return "RightHandRing2"; - case RightHandRing3: return "RightHandRing3"; - case RightInHandPinky: return "RightInHandPinky"; - case RightHandPinky1: return "RightHandPinky1"; - case RightHandPinky2: return "RightHandPinky2"; - case RightHandPinky3: return "RightHandPinky3"; - case LeftShoulder: return "LeftShoulder"; - case LeftArm: return "LeftArm"; - case LeftForeArm: return "LeftForeArm"; - case LeftHand: return "LeftHand"; - case LeftHandThumb1: return "LeftHandThumb1"; - case LeftHandThumb2: return "LeftHandThumb2"; - case LeftHandThumb3: return "LeftHandThumb3"; - case LeftInHandIndex: return "LeftInHandIndex"; - case LeftHandIndex1: return "LeftHandIndex1"; - case LeftHandIndex2: return "LeftHandIndex2"; - case LeftHandIndex3: return "LeftHandIndex3"; - case LeftInHandMiddle: return "LeftInHandMiddle"; - case LeftHandMiddle1: return "LeftHandMiddle1"; - case LeftHandMiddle2: return "LeftHandMiddle2"; - case LeftHandMiddle3: return "LeftHandMiddle3"; - case LeftInHandRing: return "LeftInHandRing"; - case LeftHandRing1: return "LeftHandRing1"; - case LeftHandRing2: return "LeftHandRing2"; - case LeftHandRing3: return "LeftHandRing3"; - case LeftInHandPinky: return "LeftInHandPinky"; - case LeftHandPinky1: return "LeftHandPinky1"; - case LeftHandPinky2: return "LeftHandPinky2"; - case LeftHandPinky3: return "LeftHandPinky3"; - default: return "???"; - } -} - controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const { - // TODO: static controller::Input::NamedVector availableInputs; - if (availableInputs.size() == 0) { - for (int i = 0; i < JointIndex::Size; i++) { - availableInputs.push_back(makePair(neuronJointIndexToPoseIndex((JointIndex)i), neuronJointName((JointIndex)i))); + for (int i = 0; i < controller::NUM_STANDARD_POSES; i++) { + auto channel = static_cast(i); + availableInputs.push_back(makePair(channel, controllerJointName(channel))); } }; return availableInputs; @@ -373,21 +465,21 @@ QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector& joints, const std::vector& prevJoints) { for (int i = 0; i < joints.size(); i++) { - int poseIndex = neuronJointIndexToPoseIndex((JointIndex)i); glm::vec3 linearVel, angularVel; glm::vec3 pos = (joints[i].pos * METERS_PER_CENTIMETER); glm::quat rot = eulerToQuat(joints[i].euler); if (i < prevJoints.size()) { - linearVel = (pos - (prevJoints[i].pos * METERS_PER_CENTIMETER)) / deltaTime; - // quat log imag part points along the axis of rotation, and it's length will be the half angle. + linearVel = (pos - (prevJoints[i].pos * METERS_PER_CENTIMETER)) / deltaTime; // m/s + // quat log imaginary part points along the axis of rotation, with length of one half the angle of rotation. glm::quat d = glm::log(rot * glm::inverse(eulerToQuat(prevJoints[i].euler))); - angularVel = glm::vec3(d.x, d.y, d.z) / (0.5f * deltaTime); + angularVel = glm::vec3(d.x, d.y, d.z) / (0.5f * deltaTime); // radians/s } + int poseIndex = neuronJointIndexToPoseIndex((NeuronJointIndex)i); _poseStateMap[poseIndex] = controller::Pose(pos, rot, linearVel, angularVel); } -} -void NeuronPlugin::InputDevice::focusOutEvent() { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::InputDevice::focusOutEvent"; + // fill in missing thumb joints with identity. + // TODO: need a standard displacement + _poseStateMap[controller::RIGHT_HAND_THUMB1] = controller::Pose(glm::vec3(), glm::quat(), glm::vec3(), glm::vec3()); + _poseStateMap[controller::LEFT_HAND_THUMB1] = controller::Pose(glm::vec3(), glm::quat(), glm::vec3(), glm::vec3()); } diff --git a/plugins/hifiNeuron/src/NeuronPlugin.h b/plugins/hifiNeuron/src/NeuronPlugin.h index f787838ce2..c85a5dd383 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.h +++ b/plugins/hifiNeuron/src/NeuronPlugin.h @@ -57,7 +57,7 @@ protected: virtual controller::Input::NamedVector getAvailableInputs() const override; virtual QString getDefaultMappingConfig() const override; virtual void update(float deltaTime, bool jointsCaptured) override {}; - virtual void focusOutEvent() override; + virtual void focusOutEvent() override {}; void update(float deltaTime, const std::vector& joints, const std::vector& prevJoints); }; @@ -71,9 +71,13 @@ protected: int _serverPort; void* _socketRef; - std::vector _joints; - std::mutex _jointsMutex; // used to guard access to _joints + // used to guard multi-threaded access to _joints + std::mutex _jointsMutex; + // copy of data directly from the NeuronDataReader SDK + std::vector _joints; + + // one frame old copy of _joints, used to caluclate angular and linear velocity. std::vector _prevJoints; }; From fd3eed3d4132e58ff5e58284ac57e1feb9ad9ae1 Mon Sep 17 00:00:00 2001 From: Anthony Thibault Date: Sat, 26 Dec 2015 11:50:54 -0800 Subject: [PATCH 07/13] NeruonPlugin: translation support Warn if translations are not present. --- examples/controllers/neuron/neuronAvatar.js | 10 +- plugins/hifiNeuron/src/NeuronPlugin.cpp | 100 +++++++++++++++----- 2 files changed, 85 insertions(+), 25 deletions(-) diff --git a/examples/controllers/neuron/neuronAvatar.js b/examples/controllers/neuron/neuronAvatar.js index cc53987f05..a124316f24 100644 --- a/examples/controllers/neuron/neuronAvatar.js +++ b/examples/controllers/neuron/neuronAvatar.js @@ -62,6 +62,8 @@ var JOINT_PARENT_MAP = { LeftHandPinky: "LeftHandPinky3", }; +var USE_TRANSLATIONS = false; + function dumpHardwareMapping() { Object.keys(Controller.Hardware).forEach(function (deviceName) { Object.keys(Controller.Hardware[deviceName]).forEach(function (input) { @@ -152,8 +154,12 @@ NeuronAvatar.prototype.update = function (deltaTime) { // Then we transform back into joint local by multiplying by the inverse of the parents absolute rotation. MyAvatar.setJointRotation(j, Quat.multiply(Quat.inverse(parentDefaultAbsRot), Quat.multiply(pose.rotation, defaultAbsRot))); - // TODO: - //MyAvatar.setJointTranslation(j, Vec3.multiply(100.0, pose.translation)); + // translation proportions might be different from the neuron avatar and the user avatar skeleton. + // so this is disabled by default + if (USE_TRANSLATIONS) { + var localTranslation = Vec3.multiplyQbyV(Quat.inverse(parentDefaultAbsRot), pose.translation); + MyAvatar.setJointTranslation(j, localTranslation); + } } } }; diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp index 4edda64c22..0e338c30f8 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.cpp +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -159,6 +159,73 @@ static controller::StandardPoseChannel neuronJointIndexToPoseIndexMap[NeuronJoin controller::LEFT_HAND_PINKY4 }; +// in rig frame +static glm::vec3 rightHandThumb1DefaultAbsTranslation(-2.155500650405884, -0.7610001564025879, 2.685631036758423); +static glm::vec3 leftHandThumb1DefaultAbsTranslation(2.1555817127227783, -0.7603635787963867, 2.6856393814086914); + +// default translations (cm) +static glm::vec3 neuronJointTranslations[NeuronJointIndex::Size] = { + {131.901, 95.6602, -27.9815}, + {-9.55907, -1.58772, 0.0760284}, + {0.0144232, -41.4683, -0.105322}, + {1.59348, -41.5875, -0.557237}, + {9.72077, -1.68926, -0.280643}, + {0.0886684, -43.1586, -0.0111596}, + {-2.98473, -44.0517, 0.0694456}, + {0.110967, 16.3959, 0.140463}, + {0.0500451, 10.0238, 0.0731921}, + {0.061568, 10.4352, 0.0583075}, + {0.0500606, 10.0217, 0.0711083}, + {0.0317731, 10.7176, 0.0779325}, + {-0.0204253, 9.71067, 0.131734}, + {-3.24245, 7.13584, 0.185638}, + {-13.0885, -0.0877601, 0.176065}, + {-27.2674, 0.0688724, 0.0272146}, + {-26.7673, 0.0301916, 0.0102847}, + {-2.56017, 0.195537, 3.20968}, + {-3.78796, 0, 0}, + {-2.63141, 0, 0}, + {-3.31579, 0.522947, 2.03495}, + {-5.36589, -0.0939789, 1.02771}, + {-3.72278, 0, 0}, + {-2.11074, 0, 0}, + {-3.47874, 0.532042, 0.778358}, + {-5.32194, -0.0864, 0.322863}, + {-4.06232, 0, 0}, + {-2.54653, 0, 0}, + {-3.46131, 0.553263, -0.132632}, + {-4.76716, -0.0227368, -0.492632}, + {-3.54073, 0, 0}, + {-2.45634, 0, 0}, + {-3.25137, 0.482779, -1.23613}, + {-4.25937, -0.0227368, -1.12168}, + {-2.83528, 0, 0}, + {-1.79166, 0, 0}, + {3.25624, 7.13148, -0.131575}, + {13.149, -0.052598, -0.125076}, + {27.2903, 0.00282644, -0.0181535}, + {26.6602, 0.000969969, -0.0487599}, + {2.56017, 0.195537, 3.20968}, + {3.78796, 0, 0}, + {2.63141, 0, 0}, + {3.31579, 0.522947, 2.03495}, + {5.36589, -0.0939789, 1.02771}, + {3.72278, 0, 0}, + {2.11074, 0, 0}, + {3.47874, 0.532042, 0.778358}, + {5.32194, -0.0864, 0.322863}, + {4.06232, 0, 0}, + {2.54653, 0, 0}, + {3.46131, 0.553263, -0.132632}, + {4.76716, -0.0227368, -0.492632}, + {3.54073, 0, 0}, + {2.45634, 0, 0}, + {3.25137, 0.482779, -1.23613}, + {4.25937, -0.0227368, -1.12168}, + {2.83528, 0, 0}, + {1.79166, 0, 0} +}; + static controller::StandardPoseChannel neuronJointIndexToPoseIndex(NeuronJointIndex i) { assert(i >= 0 && i < NeuronJointIndex::Size); if (i >= 0 && i < NeuronJointIndex::Size) { @@ -285,31 +352,20 @@ void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); } else { + qCWarning(inputplugins) << "NeuronPlugin: unsuported binary format, please enable displacements"; + // enter mutex std::lock_guard guard(neuronPlugin->_jointsMutex); - // - // Data is 3 floats per joint: 3 rotation euler angles (degrees) - // - - // resize vector if necessary - const size_t NUM_FLOATS_PER_JOINT = 3; - const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; - if (neuronPlugin->_joints.size() != NUM_JOINTS) { - neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + if (neuronPlugin->_joints.size() != NeuronJointIndex::Size) { + neuronPlugin->_joints.resize(NeuronJointIndex::Size, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); } - assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); - - for (int i = 0; i < NUM_JOINTS; i++) { - // TODO: should probably just copy over default positions?! - memcpy(&(neuronPlugin->_joints[i].euler), data, sizeof(float) * NUM_FLOATS_PER_JOINT); + for (int i = 0; i < NeuronJointIndex::Size; i++) { + neuronPlugin->_joints[i].euler = glm::vec3(); + neuronPlugin->_joints[i].pos = neuronJointTranslations[i]; } - - qCCritical(inputplugins) << "NeruonPlugin: format not supported"; - } - } else { static bool ONCE = false; if (!ONCE) { @@ -466,7 +522,7 @@ QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector& joints, const std::vector& prevJoints) { for (int i = 0; i < joints.size(); i++) { glm::vec3 linearVel, angularVel; - glm::vec3 pos = (joints[i].pos * METERS_PER_CENTIMETER); + glm::vec3 pos = joints[i].pos; glm::quat rot = eulerToQuat(joints[i].euler); if (i < prevJoints.size()) { linearVel = (pos - (prevJoints[i].pos * METERS_PER_CENTIMETER)) / deltaTime; // m/s @@ -478,8 +534,6 @@ void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector Date: Sat, 26 Dec 2015 12:13:29 -0800 Subject: [PATCH 08/13] NeuronPlugin: register for combination mode This will tell us, if user is using arms, upper-body or full body configurations. --- plugins/hifiNeuron/src/NeuronPlugin.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp index 0e338c30f8..97131a0a87 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.cpp +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -463,6 +463,8 @@ void NeuronPlugin::activate() { qCCritical(inputplugins) << "NeuronPlugin: error connecting to " << _serverAddress.c_str() << ":" << _serverPort << ", error = " << BRGetLastErrorMessage(); } else { qCDebug(inputplugins) << "NeuronPlugin: success connecting to " << _serverAddress.c_str() << ":" << _serverPort; + + BRRegisterAutoSyncParmeter(_socketRef, Cmd_CombinationMode); } } @@ -474,6 +476,7 @@ void NeuronPlugin::deactivate() { } if (_socketRef) { + BRUnregisterAutoSyncParmeter(_socketRef, Cmd_CombinationMode); BRCloseSocket(_socketRef); } From 1f834a9c01f3343368cc89f1df87c4f1e7e6c957 Mon Sep 17 00:00:00 2001 From: Anthony Thibault Date: Sun, 27 Dec 2015 16:29:30 -0800 Subject: [PATCH 09/13] neruonAvatar.js: attempt to adjust hips for HMD mode. It attempts to adjust the hips so that the avatar's head is at the same location & orientation as the HMD in world space, however it's fighting with the internal c++ code that also attempts to adjust the hips as well. --- examples/controllers/neuron/neuronAvatar.js | 78 ++++++++++++++++++--- 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/examples/controllers/neuron/neuronAvatar.js b/examples/controllers/neuron/neuronAvatar.js index a124316f24..abd51f2990 100644 --- a/examples/controllers/neuron/neuronAvatar.js +++ b/examples/controllers/neuron/neuronAvatar.js @@ -64,6 +64,27 @@ var JOINT_PARENT_MAP = { var USE_TRANSLATIONS = false; +// ctor +function Xform(rot, pos) { + this.rot = rot; + this.pos = pos; +}; +Xform.mul = function (lhs, rhs) { + var rot = Quat.multiply(lhs.rot, rhs.rot); + var pos = Vec3.sum(lhs.pos, Vec3.multiplyQbyV(lhs.rot, rhs.pos)); + return new Xform(rot, pos); +}; +Xform.prototype.inv = function () { + var invRot = Quat.inverse(this.rot); + var invPos = Vec3.multiply(-1, this.pos); + return new Xform(invRot, Vec3.multiplyQbyV(invRot, invPos)); +}; +Xform.prototype.toString = function () { + var rot = this.rot; + var pos = this.pos; + return "Xform rot = (" + rot.x + ", " + rot.y + ", " + rot.z + ", " + rot.w + "), pos = (" + pos.x + ", " + pos.y + ", " + pos.z + ")"; +}; + function dumpHardwareMapping() { Object.keys(Controller.Hardware).forEach(function (deviceName) { Object.keys(Controller.Hardware[deviceName]).forEach(function (input) { @@ -108,19 +129,19 @@ NeuronAvatar.prototype.activate = function () { this._active = true; // build absDefaultPoseMap - this._absDefaultRotMap = {}; - this._absDefaultRotMap[""] = {x: 0, y: 0, z: 0, w: 1}; + this._defaultAbsRotMap = {}; + this._defaultAbsPosMap = {}; + this._defaultAbsRotMap[""] = {x: 0, y: 0, z: 0, w: 1}; + this._defaultAbsPosMap[""] = {x: 0, y: 0, z: 0}; var keys = Object.keys(JOINT_PARENT_MAP); var i, l = keys.length; for (i = 0; i < l; i++) { var jointName = keys[i]; var j = MyAvatar.getJointIndex(jointName); var parentJointName = JOINT_PARENT_MAP[jointName]; - if (parentJointName === "") { - this._absDefaultRotMap[jointName] = MyAvatar.getDefaultJointRotation(j); - } else { - this._absDefaultRotMap[jointName] = Quat.multiply(this._absDefaultRotMap[parentJointName], MyAvatar.getDefaultJointRotation(j)); - } + this._defaultAbsRotMap[jointName] = Quat.multiply(this._defaultAbsRotMap[parentJointName], MyAvatar.getDefaultJointRotation(j)); + this._defaultAbsPosMap[jointName] = Vec3.sum(this._defaultAbsPosMap[parentJointName], + Quat.multiply(this._defaultAbsRotMap[parentJointName], MyAvatar.getDefaultJointTranslation(j))); } }; @@ -134,10 +155,16 @@ NeuronAvatar.prototype.deactivate = function () { }; NeuronAvatar.prototype.update = function (deltaTime) { + + var hmdActive = HMD.active; + var hmdXform = new Xform(HMD.orientation, HMD.position); + var keys = Object.keys(JOINT_PARENT_MAP); var i, l = keys.length; var absDefaultRot = {}; var jointName, channel, pose, parentJointName, j, parentDefaultAbsRot; + var localRotations = {}; + var localTranslations = {}; for (i = 0; i < l; i++) { var jointName = keys[i]; var channel = Controller.Hardware.Neuron[jointName]; @@ -145,23 +172,54 @@ NeuronAvatar.prototype.update = function (deltaTime) { pose = Controller.getPoseValue(channel); parentJointName = JOINT_PARENT_MAP[jointName]; j = MyAvatar.getJointIndex(jointName); - defaultAbsRot = this._absDefaultRotMap[jointName]; - parentDefaultAbsRot = this._absDefaultRotMap[parentJointName]; + defaultAbsRot = this._defaultAbsRotMap[jointName]; + parentDefaultAbsRot = this._defaultAbsRotMap[parentJointName]; // Rotations from the neuron controller are in world orientation but are delta's from the default pose. // So first we build the absolute rotation of the default pose (local into world). // Then apply the rotation from the controller, in world space. // Then we transform back into joint local by multiplying by the inverse of the parents absolute rotation. - MyAvatar.setJointRotation(j, Quat.multiply(Quat.inverse(parentDefaultAbsRot), Quat.multiply(pose.rotation, defaultAbsRot))); + var localRotation = Quat.multiply(Quat.inverse(parentDefaultAbsRot), Quat.multiply(pose.rotation, defaultAbsRot)); + if (!hmdActive || jointName !== "Hips") { + MyAvatar.setJointRotation(j, localRotation); + } + localRotations[jointName] = localRotation; // translation proportions might be different from the neuron avatar and the user avatar skeleton. // so this is disabled by default if (USE_TRANSLATIONS) { var localTranslation = Vec3.multiplyQbyV(Quat.inverse(parentDefaultAbsRot), pose.translation); MyAvatar.setJointTranslation(j, localTranslation); + localTranslations[jointName] = localTranslation; + } else { + localTranslations[jointName] = MyAvatar.getJointTranslation(j); } } } + + // TODO: Currrently does not work. + // it attempts to adjust the hips so that the avatar's head is at the same location & oreintation as the HMD. + // however it's fighting with the internal c++ code that also attempts to adjust the hips. + if (hmdActive) { + + var y180Xform = new Xform({x: 0, y: 1, z: 0, w: 0}, {x: 0, y: 0, z: 0}); + var avatarXform = new Xform(MyAvatar.orientation, MyAvatar.position); + var headXform = new Xform(localRotations["Head"], localTranslations["Head"]); + + // transform eyes down the heirarchy chain into avatar space. + var hierarchy = ["Neck", "Spine3", "Spine2", "Spine1", "Spine"]; + var i, l = hierarchy.length; + for (i = 0; i < l; i++) { + var xform = new Xform(localRotations[hierarchy[i]], localTranslations[hierarchy[i]]); + headXform = Xform.mul(xform, headXform); + } + + // solve for the offset that will put the eyes at the hmd position & orientation. + var hipsXform = Xform.mul(y180Xform, Xform.mul(avatarXform.inv(), Xform.mul(hmdXform, Xform.mul(y180Xform, headXform.inv())))); + + MyAvatar.setJointRotation("Hips", hipsXform.rot); + MyAvatar.setJointTranslation("Hips", hipsXform.pos); + } }; var neuronAvatar = new NeuronAvatar(); From 4661152aa779c0e6dc1957e467f0fe237188816b Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Mon, 28 Dec 2015 17:57:24 -0800 Subject: [PATCH 10/13] NeuronAvatar.js: Now aligns the head with the HMD if active --- examples/controllers/neuron/neuronAvatar.js | 23 ++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/examples/controllers/neuron/neuronAvatar.js b/examples/controllers/neuron/neuronAvatar.js index abd51f2990..7cebc13feb 100644 --- a/examples/controllers/neuron/neuronAvatar.js +++ b/examples/controllers/neuron/neuronAvatar.js @@ -157,8 +157,6 @@ NeuronAvatar.prototype.deactivate = function () { NeuronAvatar.prototype.update = function (deltaTime) { var hmdActive = HMD.active; - var hmdXform = new Xform(HMD.orientation, HMD.position); - var keys = Object.keys(JOINT_PARENT_MAP); var i, l = keys.length; var absDefaultRot = {}; @@ -192,18 +190,22 @@ NeuronAvatar.prototype.update = function (deltaTime) { MyAvatar.setJointTranslation(j, localTranslation); localTranslations[jointName] = localTranslation; } else { - localTranslations[jointName] = MyAvatar.getJointTranslation(j); + localTranslations[jointName] = MyAvatar.getDefaultJointTranslation(j); } } } - // TODO: Currrently does not work. // it attempts to adjust the hips so that the avatar's head is at the same location & oreintation as the HMD. // however it's fighting with the internal c++ code that also attempts to adjust the hips. if (hmdActive) { - + var UNIT_SCALE = 1 / 100; + var hmdXform = new Xform(HMD.orientation, Vec3.multiply(1 / UNIT_SCALE, HMD.position)); // convert to cm var y180Xform = new Xform({x: 0, y: 1, z: 0, w: 0}, {x: 0, y: 0, z: 0}); - var avatarXform = new Xform(MyAvatar.orientation, MyAvatar.position); + var avatarXform = new Xform(MyAvatar.orientation, Vec3.multiply(1 / UNIT_SCALE, MyAvatar.position)); // convert to cm + var hipsJointIndex = MyAvatar.getJointIndex("Hips"); + var modelOffsetInvXform = new Xform({x: 0, y: 0, z: 0, w: 1}, MyAvatar.getDefaultJointTranslation(hipsJointIndex)); + var defaultHipsXform = new Xform(MyAvatar.getDefaultJointRotation(hipsJointIndex), MyAvatar.getDefaultJointTranslation(hipsJointIndex)); + var headXform = new Xform(localRotations["Head"], localTranslations["Head"]); // transform eyes down the heirarchy chain into avatar space. @@ -213,9 +215,16 @@ NeuronAvatar.prototype.update = function (deltaTime) { var xform = new Xform(localRotations[hierarchy[i]], localTranslations[hierarchy[i]]); headXform = Xform.mul(xform, headXform); } + headXform = Xform.mul(defaultHipsXform, headXform); + + var preXform = Xform.mul(headXform, y180Xform); + var postXform = Xform.mul(avatarXform, Xform.mul(y180Xform, modelOffsetInvXform.inv())); // solve for the offset that will put the eyes at the hmd position & orientation. - var hipsXform = Xform.mul(y180Xform, Xform.mul(avatarXform.inv(), Xform.mul(hmdXform, Xform.mul(y180Xform, headXform.inv())))); + var hipsOffsetXform = Xform.mul(postXform.inv(), Xform.mul(hmdXform, preXform.inv())); + + // now combine it with the default hips transform + var hipsXform = Xform.mul(hipsOffsetXform, defaultHipsXform); MyAvatar.setJointRotation("Hips", hipsXform.rot); MyAvatar.setJointTranslation("Hips", hipsXform.pos); From 7e514d2f4dd477356d53ffff6c362549bea7de6d Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Mon, 28 Dec 2015 18:42:03 -0800 Subject: [PATCH 11/13] Mac build fix --- cmake/externals/neuron/CMakeLists.txt | 9 ++++++++- plugins/hifiNeuron/src/NeuronPlugin.cpp | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cmake/externals/neuron/CMakeLists.txt b/cmake/externals/neuron/CMakeLists.txt index 324b3fb917..6936725571 100644 --- a/cmake/externals/neuron/CMakeLists.txt +++ b/cmake/externals/neuron/CMakeLists.txt @@ -41,8 +41,15 @@ if(WIN32) set(${EXTERNAL_NAME_UPPER}_LIBRARIES "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.lib" CACHE TYPE INTERNAL) add_paths_to_fixup_libs("${${EXTERNAL_NAME_UPPER}_LIB_PATH}") + elseif(APPLE) - # TODO + + set(${EXTERNAL_NAME_UPPER}_LIB_PATH "${SOURCE_DIR}/NeuronDataReader_Mac/dylib") + set(${EXTERNAL_NAME_UPPER}_LIBRARY_RELEASE "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.dylib" CACHE TYPE INTERNAL) + set(${EXTERNAL_NAME_UPPER}_LIBRARIES "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.dylib" CACHE TYPE INTERNAL) + + add_paths_to_fixup_libs("${${EXTERNAL_NAME_UPPER}_LIB_PATH}") + else() # UNSUPPORTED endif() diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp index 97131a0a87..6132c16a43 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.cpp +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -523,7 +523,7 @@ QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { } void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector& joints, const std::vector& prevJoints) { - for (int i = 0; i < joints.size(); i++) { + for (size_t i = 0; i < joints.size(); i++) { glm::vec3 linearVel, angularVel; glm::vec3 pos = joints[i].pos; glm::quat rot = eulerToQuat(joints[i].euler); From c35995b8e3363dda1ec0f129db5e6423cfd4c408 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 29 Dec 2015 08:51:09 -0800 Subject: [PATCH 12/13] Build fix for linux? --- cmake/macros/TargetNeuron.cmake | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmake/macros/TargetNeuron.cmake b/cmake/macros/TargetNeuron.cmake index 01891ef525..11a92e68ad 100644 --- a/cmake/macros/TargetNeuron.cmake +++ b/cmake/macros/TargetNeuron.cmake @@ -6,9 +6,12 @@ # See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html # macro(TARGET_NEURON) - add_dependency_external_projects(neuron) - find_package(Neuron REQUIRED) - target_include_directories(${TARGET_NAME} PRIVATE ${NEURON_INCLUDE_DIRS}) - target_link_libraries(${TARGET_NAME} ${NEURON_LIBRARIES}) - add_definitions(-DHAVE_NEURON) + # Neuron data reader is only available on these platforms + if (WIN32 OR APPLE) + add_dependency_external_projects(neuron) + find_package(Neuron REQUIRED) + target_include_directories(${TARGET_NAME} PRIVATE ${NEURON_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${NEURON_LIBRARIES}) + add_definitions(-DHAVE_NEURON) + endif(WIN32 OR APPLE) endmacro() From e10cecd3101400d2228db6b3f5a9d195f357193c Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 29 Dec 2015 09:23:03 -0800 Subject: [PATCH 13/13] Build fix for linux #2? --- plugins/hifiNeuron/src/NeuronPlugin.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp index 6132c16a43..a175ce8e06 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.cpp +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -24,7 +24,10 @@ Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") #define __OS_XUN__ 1 #define BOOL int + +#ifdef HAVE_NEURON #include +#endif const QString NeuronPlugin::NAME = "Neuron"; const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; @@ -312,6 +315,8 @@ static quat eulerToQuat(vec3 euler) { glm::angleAxis(e.z, Vectors::UNIT_Z)); } +#ifdef HAVE_NEURON + // // neuronDataReader SDK callback functions // @@ -430,17 +435,24 @@ static void SocketStatusChangedCallback(void* context, SOCKET_REF sender, Socket qCDebug(inputplugins) << "NeuronPlugin: socket status = " << message; } +#endif // #ifdef HAVE_NEURON + // // NeuronPlugin // bool NeuronPlugin::isSupported() const { +#ifdef HAVE_NEURON // Because it's a client/server network architecture, we can't tell // if the neuron is actually connected until we connect to the server. return true; +#else + return false; +#endif } void NeuronPlugin::activate() { +#ifdef HAVE_NEURON InputPlugin::activate(); // register with userInputMapper @@ -466,9 +478,11 @@ void NeuronPlugin::activate() { BRRegisterAutoSyncParmeter(_socketRef, Cmd_CombinationMode); } +#endif } void NeuronPlugin::deactivate() { +#ifdef HAVE_NEURON // unregister from userInputMapper if (_inputDevice->_deviceID != controller::Input::INVALID_DEVICE) { auto userInputMapper = DependencyManager::get(); @@ -481,6 +495,7 @@ void NeuronPlugin::deactivate() { } InputPlugin::deactivate(); +#endif } void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) {