From 0459479c2b1a8130beecee5771d40aac490d1a9f Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Mon, 21 Dec 2015 18:30:15 -0800 Subject: [PATCH] NeuronPlugin: Added external project for Neuron SDK Now builds on windows with actual Neuron SDK. Can to TCP server on localhost, and receive joint data. Will debug draw joint 6, (left foot?) --- cmake/externals/neuron/CMakeLists.txt | 49 ++++ cmake/macros/TargetNeuron.cmake | 14 ++ cmake/modules/FindNeuron.cmake | 28 +++ interface/CMakeLists.txt | 4 +- plugins/{neuron => hifiNeuron}/CMakeLists.txt | 4 +- plugins/hifiNeuron/src/NeuronPlugin.cpp | 222 ++++++++++++++++++ .../{neuron => hifiNeuron}/src/NeuronPlugin.h | 19 +- .../src/NeuronProvider.cpp | 0 .../{neuron => hifiNeuron}/src/plugin.json | 0 plugins/neuron/src/NeuronPlugin.cpp | 75 ------ 10 files changed, 336 insertions(+), 79 deletions(-) create mode 100644 cmake/externals/neuron/CMakeLists.txt create mode 100644 cmake/macros/TargetNeuron.cmake create mode 100644 cmake/modules/FindNeuron.cmake rename plugins/{neuron => hifiNeuron}/CMakeLists.txt (88%) create mode 100644 plugins/hifiNeuron/src/NeuronPlugin.cpp rename plugins/{neuron => hifiNeuron}/src/NeuronPlugin.h (79%) rename plugins/{neuron => hifiNeuron}/src/NeuronProvider.cpp (100%) rename plugins/{neuron => hifiNeuron}/src/plugin.json (100%) delete mode 100644 plugins/neuron/src/NeuronPlugin.cpp diff --git a/cmake/externals/neuron/CMakeLists.txt b/cmake/externals/neuron/CMakeLists.txt new file mode 100644 index 0000000000..324b3fb917 --- /dev/null +++ b/cmake/externals/neuron/CMakeLists.txt @@ -0,0 +1,49 @@ +include(ExternalProject) + +set(EXTERNAL_NAME neuron) + +string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) + +set(NEURON_URL "https://s3.amazonaws.com/hifi-public/dependencies/neuron_datareader_b.12.zip") +set(NEURON_URL_MD5 "0ab54ca04c9cc8094e0fa046c226e574") + +ExternalProject_Add(${EXTERNAL_NAME} + URL ${NEURON_URL} + URL_MD5 ${NEURON_URL_MD5} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + LOG_DOWNLOAD 1) + +ExternalProject_Get_Property(${EXTERNAL_NAME} SOURCE_DIR) + +set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals") + +# set include dir +if(WIN32) + set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS "${SOURCE_DIR}/NeuronDataReader_Windows/include" CACHE TYPE INTERNAL) +elseif(APPLE) + set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS "${SOURCE_DIR}/NeuronDataReader_Mac/include" CACHE TYPE INTERNAL) +else() + # Unsupported +endif() + +if(WIN32) + + if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + set(ARCH_DIR "x64") + else() + set(ARCH_DIR "x86") + endif() + + set(${EXTERNAL_NAME_UPPER}_LIB_PATH "${SOURCE_DIR}/NeuronDataReader_Windows/lib/${ARCH_DIR}") + set(${EXTERNAL_NAME_UPPER}_LIBRARY_RELEASE "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.lib" CACHE TYPE INTERNAL) + set(${EXTERNAL_NAME_UPPER}_LIBRARIES "${${EXTERNAL_NAME_UPPER}_LIB_PATH}/NeuronDataReader.lib" CACHE TYPE INTERNAL) + + add_paths_to_fixup_libs("${${EXTERNAL_NAME_UPPER}_LIB_PATH}") +elseif(APPLE) + # TODO +else() + # UNSUPPORTED +endif() + diff --git a/cmake/macros/TargetNeuron.cmake b/cmake/macros/TargetNeuron.cmake new file mode 100644 index 0000000000..01891ef525 --- /dev/null +++ b/cmake/macros/TargetNeuron.cmake @@ -0,0 +1,14 @@ +# +# Copyright 2015 High Fidelity, Inc. +# Created by Anthony J. Thibault on 2015/12/21 +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# +macro(TARGET_NEURON) + add_dependency_external_projects(neuron) + find_package(Neuron REQUIRED) + target_include_directories(${TARGET_NAME} PRIVATE ${NEURON_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${NEURON_LIBRARIES}) + add_definitions(-DHAVE_NEURON) +endmacro() diff --git a/cmake/modules/FindNeuron.cmake b/cmake/modules/FindNeuron.cmake new file mode 100644 index 0000000000..6a93b3a016 --- /dev/null +++ b/cmake/modules/FindNeuron.cmake @@ -0,0 +1,28 @@ +# +# FindNeuron.cmake +# +# Try to find the Perception Neuron SDK +# +# You must provide a NEURON_ROOT_DIR which contains lib and include directories +# +# Once done this will define +# +# NEURON_FOUND - system found Neuron SDK +# NEURON_INCLUDE_DIRS - the Neuron SDK include directory +# NEURON_LIBRARIES - Link this to use Neuron +# +# Created on 12/21/2015 by Anthony J. Thibault +# Copyright 2015 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# + +include(SelectLibraryConfigurations) +select_library_configurations(NEURON) + +set(NEURON_REQUIREMENTS NEURON_INCLUDE_DIRS NEURON_LIBRARIES) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Neuron DEFAULT_MSG NEURON_INCLUDE_DIRS NEURON_LIBRARIES) +mark_as_advanced(NEURON_LIBRARIES NEURON_INCLUDE_DIRS NEURON_SEARCH_DIRS) + diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 1d9557a835..5d96b95624 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -109,7 +109,9 @@ add_dependency_external_projects(sdl2) if (WIN32) add_dependency_external_projects(OpenVR) endif() - +if(WIN32 OR APPLE) + add_dependency_external_projects(neuron) +endif() # disable /OPT:REF and /OPT:ICF for the Debug builds # This will prevent the following linker warnings diff --git a/plugins/neuron/CMakeLists.txt b/plugins/hifiNeuron/CMakeLists.txt similarity index 88% rename from plugins/neuron/CMakeLists.txt rename to plugins/hifiNeuron/CMakeLists.txt index b86d310ab7..9c512fc877 100644 --- a/plugins/neuron/CMakeLists.txt +++ b/plugins/hifiNeuron/CMakeLists.txt @@ -6,8 +6,8 @@ # See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html # -set(TARGET_NAME neuron) +set(TARGET_NAME hifiNeuron) setup_hifi_plugin(Script Qml Widgets) link_hifi_libraries(shared controllers plugins input-plugins) -# target_neuron() +target_neuron() diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp new file mode 100644 index 0000000000..c3f764da05 --- /dev/null +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -0,0 +1,222 @@ +// +// NeuronPlugin.h +// input-plugins/src/input-plugins +// +// Created by Anthony Thibault on 12/18/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "NeuronPlugin.h" + +#include +#include +#include +#include +#include + +Q_DECLARE_LOGGING_CATEGORY(inputplugins) +Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") + +#define __OS_XUN__ 1 +#define BOOL int +#include + +const QString NeuronPlugin::NAME = "Neuron"; +const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; + +enum JointIndex { + HipsPosition = 0, + Hips, + RightUpLeg, + RightLeg, + RightFoot, + LeftUpLeg, + LeftLeg, + LeftFoot, + Spine, + Spine1, + Spine2, + Spine3, + Neck, + Head, + RightShoulder, + RightArm, + RightForeArm, + RightHand, + RightHandThumb1, + RightHandThumb2, + RightHandThumb3, + RightInHandIndex, + RightHandIndex1, + RightHandIndex2, + RightHandIndex3, + RightInHandMiddle, + RightHandMiddle1, + RightHandMiddle2, + RightHandMiddle3, + RightInHandRing, + RightHandRing1, + RightHandRing2, + RightHandRing3, + RightInHandPinky, + RightHandPinky1, + RightHandPinky2, + RightHandPinky3, + LeftShoulder, + LeftArm, + LeftForeArm, + LeftHand, + LeftHandThumb1, + LeftHandThumb2, + LeftHandThumb3, + LeftInHandIndex, + LeftHandIndex1, + LeftHandIndex2, + LeftHandIndex3, + LeftInHandMiddle, + LeftHandMiddle1, + LeftHandMiddle2, + LeftHandMiddle3, + LeftInHandRing, + LeftHandRing1, + LeftHandRing2, + LeftHandRing3, + LeftInHandPinky, + LeftHandPinky1, + LeftHandPinky2, + LeftHandPinky3 +}; + +bool NeuronPlugin::isSupported() const { + // Because it's a client/server network architecture, we can't tell + // if the neuron is actually connected until we connect to the server. + return true; +} + +// NOTE: must be thread-safe +void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx* header, float* data) { + qCDebug(inputplugins) << "NeuronPlugin: received frame data, DataCount = " << header->DataCount; + + auto neuronPlugin = reinterpret_cast(context); + std::lock_guard guard(neuronPlugin->_jointsMutex); + + // Data is 6 floats: 3 position values, 3 rotation euler angles (degrees) + + // resize vector if necessary + const size_t NUM_FLOATS_PER_JOINT = 6; + const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; + if (neuronPlugin->_joints.size() != NUM_JOINTS) { + neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + } + + assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); + + // copy the data + memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); +} + +// NOTE: must be thread-safe +static void CommandDataReceivedCallback(void* context, SOCKET_REF sender, CommandPack* pack, void* data) { + +} + +// NOTE: must be thread-safe +static void SocketStatusChangedCallback(void* context, SOCKET_REF sender, SocketStatus status, char* message) { + qCDebug(inputplugins) << "NeuronPlugin: socket status = " << message; +} + +void NeuronPlugin::activate() { + InputPlugin::activate(); + qCDebug(inputplugins) << "NeuronPlugin::activate"; + + // register c-style callbacks + BRRegisterFrameDataCallback((void*)this, FrameDataReceivedCallback); + BRRegisterCommandDataCallback((void*)this, CommandDataReceivedCallback); + BRRegisterSocketStatusCallback((void*)this, SocketStatusChangedCallback); + + // TODO: pull these from prefs! + _serverAddress = "localhost"; + _serverPort = 7001; + _socketRef = BRConnectTo((char*)_serverAddress.c_str(), _serverPort); + if (!_socketRef) { + // error + qCCritical(inputplugins) << "NeuronPlugin: error connecting to " << _serverAddress.c_str() << ":" << _serverPort << "error = " << BRGetLastErrorMessage(); + } + qCDebug(inputplugins) << "NeuronPlugin: success connecting to " << _serverAddress.c_str() << ":" << _serverPort; +} + +void NeuronPlugin::deactivate() { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::deactivate"; + + if (_socketRef) { + BRCloseSocket(_socketRef); + } + InputPlugin::deactivate(); +} + +// convert between euler in degrees to quaternion +static quat eulerToQuat(vec3 euler) { + return (glm::angleAxis(euler.y * RADIANS_PER_DEGREE, Vectors::UNIT_Y) * + glm::angleAxis(euler.x * RADIANS_PER_DEGREE, Vectors::UNIT_X) * + glm::angleAxis(euler.z * RADIANS_PER_DEGREE, Vectors::UNIT_Z)); +} + +void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { + + std::vector joints; + // copy the shared data + { + std::lock_guard guard(_jointsMutex); + joints = _joints; + } + + DebugDraw::getInstance().addMyAvatarMarker("LEFT_FOOT", + eulerToQuat(joints[6].rot), + joints[6].pos / 100.0f, + glm::vec4(1)); + + _inputDevice->update(deltaTime, jointsCaptured); +} + +void NeuronPlugin::saveSettings() const { + InputPlugin::saveSettings(); + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::saveSettings"; +} + +void NeuronPlugin::loadSettings() { + InputPlugin::loadSettings(); + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::loadSettings"; +} + +// +// InputDevice +// + +controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const { + // TODO: + static const controller::Input::NamedVector availableInputs { + makePair(controller::LEFT_HAND, "LeftHand"), + makePair(controller::RIGHT_HAND, "RightHand") + }; + return availableInputs; +} + +QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { + static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/neuron.json"; + return MAPPING_JSON; +} + +void NeuronPlugin::InputDevice::update(float deltaTime, bool jointsCaptured) { + +} + +void NeuronPlugin::InputDevice::focusOutEvent() { + // TODO: + qCDebug(inputplugins) << "NeuronPlugin::InputDevice::focusOutEvent"; +} diff --git a/plugins/neuron/src/NeuronPlugin.h b/plugins/hifiNeuron/src/NeuronPlugin.h similarity index 79% rename from plugins/neuron/src/NeuronPlugin.h rename to plugins/hifiNeuron/src/NeuronPlugin.h index 59e0f0a393..5f67502e04 100644 --- a/plugins/neuron/src/NeuronPlugin.h +++ b/plugins/hifiNeuron/src/NeuronPlugin.h @@ -16,10 +16,15 @@ #include #include +struct _BvhDataHeaderEx; +void FrameDataReceivedCallback(void* context, void* sender, _BvhDataHeaderEx* header, float* data); + // Handles interaction with the Neuron SDK class NeuronPlugin : public InputPlugin { Q_OBJECT public: + friend void FrameDataReceivedCallback(void* context, void* sender, _BvhDataHeaderEx* header, float* data); + // Plugin functions virtual bool isSupported() const override; virtual bool isJointController() const override { return true; } @@ -35,7 +40,7 @@ public: virtual void saveSettings() const override; virtual void loadSettings() override; -private: +protected: class InputDevice : public controller::InputDevice { public: InputDevice() : controller::InputDevice("Neuron") {} @@ -51,6 +56,18 @@ private: static const QString NAME; static const QString NEURON_ID_STRING; + + std::string _serverAddress; + int _serverPort; + void* _socketRef; + + struct NeuronJoint { + glm::vec3 pos; + glm::vec3 rot; + }; + + std::vector _joints; + std::mutex _jointsMutex; }; #endif // hifi_NeuronPlugin_h diff --git a/plugins/neuron/src/NeuronProvider.cpp b/plugins/hifiNeuron/src/NeuronProvider.cpp similarity index 100% rename from plugins/neuron/src/NeuronProvider.cpp rename to plugins/hifiNeuron/src/NeuronProvider.cpp diff --git a/plugins/neuron/src/plugin.json b/plugins/hifiNeuron/src/plugin.json similarity index 100% rename from plugins/neuron/src/plugin.json rename to plugins/hifiNeuron/src/plugin.json diff --git a/plugins/neuron/src/NeuronPlugin.cpp b/plugins/neuron/src/NeuronPlugin.cpp deleted file mode 100644 index 735b81a1ef..0000000000 --- a/plugins/neuron/src/NeuronPlugin.cpp +++ /dev/null @@ -1,75 +0,0 @@ -// -// NeuronPlugin.h -// input-plugins/src/input-plugins -// -// Created by Anthony Thibault on 12/18/2015. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "NeuronPlugin.h" - -#include -#include - -Q_DECLARE_LOGGING_CATEGORY(inputplugins) -Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") - -const QString NeuronPlugin::NAME = "Neuron"; -const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; - -bool NeuronPlugin::isSupported() const { - // TODO: - return true; -} - -void NeuronPlugin::activate() { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::activate"; -} - -void NeuronPlugin::deactivate() { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::deactivate"; -} - -void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::pluginUpdate"; -} - -void NeuronPlugin::saveSettings() const { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::saveSettings"; -} - -void NeuronPlugin::loadSettings() { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::loadSettings"; -} - -controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const { - // TODO: - static const controller::Input::NamedVector availableInputs { - makePair(controller::LEFT_HAND, "LeftHand"), - makePair(controller::RIGHT_HAND, "RightHand") - }; - return availableInputs; -} - -QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { - static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/neuron.json"; - return MAPPING_JSON; -} - -void NeuronPlugin::InputDevice::update(float deltaTime, bool jointsCaptured) { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::InputDevice::update"; -} - -void NeuronPlugin::InputDevice::focusOutEvent() { - // TODO: - qCDebug(inputplugins) << "NeuronPlugin::InputDevice::focusOutEvent"; -}