diff --git a/examples/controllers/neuron/neuronAvatar.js b/examples/controllers/neuron/neuronAvatar.js new file mode 100644 index 0000000000..a7146e0759 --- /dev/null +++ b/examples/controllers/neuron/neuronAvatar.js @@ -0,0 +1,141 @@ +// maps controller joint names to avatar joint names +var JOINT_NAME_MAP = { + HipsPosition: "", + Hips: "Hips", + RightUpLeg: "RightUpLeg", + RightLeg: "RightLeg", + RightFoot: "RightFoot", + LeftUpLeg: "LeftUpLeg", + LeftLeg: "LeftLeg", + LeftFoot: "LeftFoot", + Spine: "Spine", + Spine1: "Spine1", + Spine2: "Spine2", + Spine3: "Spine3", + Neck: "Neck", + Head: "Head", + RightShoulder: "RightShoulder", + RightArm: "RightArm", + RightForeArm: "RightForeArm", + RightHand: "RightHand", + RightHandThumb1: "RightHandThumb2", + RightHandThumb2: "RightHandThumb3", + RightHandThumb3: "RightHandThumb4", + RightInHandIndex: "RightHandIndex1", + RightHandIndex1: "RightHandIndex2", + RightHandIndex2: "RightHandIndex3", + RightHandIndex3: "RightHandIndex4", + RightInHandMiddle: "RightHandMiddle1", + RightHandMiddle1: "RightHandMiddle2", + RightHandMiddle2: "RightHandMiddle3", + RightHandMiddle3: "RightHandMiddle4", + RightInHandRing: "RightHandRing1", + RightHandRing1: "RightHandRing2", + RightHandRing2: "RightHandRing3", + RightHandRing3: "RightHandRing4", + RightInHandPinky: "RightHandPinky1", + RightHandPinky1: "RightHandPinky2", + RightHandPinky2: "RightHandPinky3", + RightHandPinky3: "RightHandPinky4", + LeftShoulder: "LeftShoulder", + LeftArm: "LeftArm", + LeftForeArm: "LeftForeArm", + LeftHand: "LeftHand", + LeftHandThumb1: "LeftHandThumb2", + LeftHandThumb2: "LeftHandThumb3", + LeftHandThumb3: "LeftHandThumb4", + LeftInHandIndex: "LeftHandIndex1", + LeftHandIndex1: "LeftHandIndex2", + LeftHandIndex2: "LeftHandIndex3", + LeftHandIndex3: "LeftHandIndex4", + LeftInHandMiddle: "LeftHandMiddle1", + LeftHandMiddle1: "LeftHandMiddle2", + LeftHandMiddle2: "LeftHandMiddle3", + LeftHandMiddle3: "LeftHandMiddle4", + LeftInHandRing: "LeftHandRing1", + LeftHandRing1: "LeftHandRing2", + LeftHandRing2: "LeftHandRing3", + LeftHandRing3: "LeftHandRing4", + LeftInHandPinky: "LeftHandPinky1", + LeftHandPinky1: "LeftHandPinky2", + LeftHandPinky2: "LeftHandPinky3", + LeftHandPinky3: "LeftHandPinky4" +}; + +function dumpHardwareMapping() { + Object.keys(Controller.Hardware).forEach(function (deviceName) { + Object.keys(Controller.Hardware[deviceName]).forEach(function (input) { + print("Controller.Hardware." + deviceName + "." + input + ":" + Controller.Hardware[deviceName][input]); + }); + }); +} + +// ctor +function NeuronAvatar() { + var self = this; + Script.scriptEnding.connect(function () { + self.shutdown(); + }); + Controller.hardwareChanged.connect(function () { + self.hardwareChanged(); + }); + + if (Controller.Hardware.Neuron) { + this.activate(); + } else { + this.deactivate(); + } +} + +NeuronAvatar.prototype.shutdown = function () { + this.deactivate(); +}; + +NeuronAvatar.prototype.hardwareChanged = function () { + if (Controller.Hardware.Neuron) { + this.activate(); + } else { + this.deactivate(); + } +}; + +NeuronAvatar.prototype.activate = function () { + if (!this._active) { + Script.update.connect(updateCallback); + } + this._active = true; +}; + +NeuronAvatar.prototype.deactivate = function () { + if (this._active) { + var self = this; + Script.update.disconnect(updateCallback); + } + this._active = false; + MyAvatar.clearJointsData(); +}; + +NeuronAvatar.prototype.update = function (deltaTime) { + var keys = Object.keys(JOINT_NAME_MAP); + var i, l = keys.length; + for (i = 0; i < l; i++) { + var channel = Controller.Hardware.Neuron[keys[i]]; + if (channel) { + var pose = Controller.getPoseValue(channel); + var j = MyAvatar.getJointIndex(JOINT_NAME_MAP[keys[i]]); + var defaultRot = MyAvatar.getDefaultJointRotation(j); + if (keys[i] == "Hips") { + MyAvatar.setJointRotation(j, Quat.multiply(pose.rotation, defaultRot)); + } else { + MyAvatar.setJointRotation(j, defaultRot); + } + } + } +}; + +var neuronAvatar = new NeuronAvatar(); + +function updateCallback(deltaTime) { + neuronAvatar.update(deltaTime); +} + diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index 68f382d2d9..4dd091f1d6 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -289,8 +289,10 @@ void Rig::clearJointState(int index) { void Rig::clearJointStates() { _internalPoseSet._overrideFlags.clear(); - _internalPoseSet._overrideFlags.resize(_animSkeleton->getNumJoints()); - _internalPoseSet._overridePoses = _animSkeleton->getRelativeDefaultPoses(); + if (_animSkeleton) { + _internalPoseSet._overrideFlags.resize(_animSkeleton->getNumJoints()); + _internalPoseSet._overridePoses = _animSkeleton->getRelativeDefaultPoses(); + } } void Rig::clearJointAnimationPriority(int index) { diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index bbd33c5cb3..feed8a0fad 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -88,9 +88,66 @@ namespace controller { // No correlation to SDL enum StandardPoseChannel { - LEFT_HAND = 0, - RIGHT_HAND, + HIPS_ROOT = 0, + HIPS, + RIGHT_UP_LEG, + RIGHT_LEG, + RIGHT_FOOT, + LEFT_UP_LEG, + LEFT_LEG, + LEFT_FOOT, + SPINE, + SPINE1, + SPINE2, + SPINE3, + NECK, HEAD, + RIGHT_SHOULDER, + RIGHT_ARM, + RIGHT_FORE_ARM, + RIGHT_HAND, + RIGHT_HAND_THUMB1, + RIGHT_HAND_THUMB2, + RIGHT_HAND_THUMB3, + RIGHT_IN_HAND_INDEX, + RIGHT_HAND_INDEX1, + RIGHT_HAND_INDEX2, + RIGHT_HAND_INDEX3, + RIGHT_IN_HAND_MIDDLE, + RIGHT_HAND_MIDDLE1, + RIGHT_HAND_MIDDLE2, + RIGHT_HAND_MIDDLE3, + RIGHT_IN_HANDRING, + RIGHT_HAND_RING1, + RIGHT_HAND_RING2, + RIGHT_HAND_RING3, + RIGHT_IN_HAND_PINKY, + RIGHT_HAND_PINKY1, + RIGHT_HAND_PINKY2, + RIGHT_HAND_PINKY3, + LEFT_SHOULDER, + LEFT_ARM, + LEFT_FORE_ARM, + LEFT_HAND, + LEFT_HAND_THUMB1, + LEFT_HAND_THUMB2, + LEFT_HAND_THUMB3, + LEFT_IN_HAND_INDEX, + LEFT_HAND_INDEX1, + LEFT_HAND_INDEX2, + LEFT_HAND_INDEX3, + LEFT_IN_HAND_MIDDLE, + LEFT_HAND_MIDDLE1, + LEFT_HAND_MIDDLE2, + LEFT_HAND_MIDDLE3, + LEFT_IN_HAND_RING, + LEFT_HAND_RING1, + LEFT_HAND_RING2, + LEFT_HAND_RING3, + LEFT_IN_HAND_PINKY, + LEFT_HAND_PINKY1, + LEFT_HAND_PINKY2, + LEFT_HAND_PINKY3, NUM_STANDARD_POSES }; diff --git a/plugins/hifiNeuron/src/NeuronPlugin.cpp b/plugins/hifiNeuron/src/NeuronPlugin.cpp index c3f764da05..4f52d8da98 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.cpp +++ b/plugins/hifiNeuron/src/NeuronPlugin.cpp @@ -1,5 +1,5 @@ // -// NeuronPlugin.h +// NeuronPlugin.cpp // input-plugins/src/input-plugins // // Created by Anthony Thibault on 12/18/2015. @@ -11,6 +11,7 @@ #include "NeuronPlugin.h" +#include #include #include #include @@ -27,6 +28,7 @@ Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins") const QString NeuronPlugin::NAME = "Neuron"; const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron"; +// This matches controller::StandardPoseChannel enum JointIndex { HipsPosition = 0, Hips, @@ -87,7 +89,8 @@ enum JointIndex { LeftInHandPinky, LeftHandPinky1, LeftHandPinky2, - LeftHandPinky3 + LeftHandPinky3, + Size }; bool NeuronPlugin::isSupported() const { @@ -98,29 +101,81 @@ bool NeuronPlugin::isSupported() const { // NOTE: must be thread-safe void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx* header, float* data) { - qCDebug(inputplugins) << "NeuronPlugin: received frame data, DataCount = " << header->DataCount; auto neuronPlugin = reinterpret_cast(context); - std::lock_guard guard(neuronPlugin->_jointsMutex); - // Data is 6 floats: 3 position values, 3 rotation euler angles (degrees) + // version 1.0 + if (header->DataVersion.Major == 1 && header->DataVersion.Minor == 0) { - // resize vector if necessary - const size_t NUM_FLOATS_PER_JOINT = 6; - const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; - if (neuronPlugin->_joints.size() != NUM_JOINTS) { - neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + std::lock_guard guard(neuronPlugin->_jointsMutex); + + // Data is 6 floats: 3 position values, 3 rotation euler angles (degrees) + + // resize vector if necessary + const size_t NUM_FLOATS_PER_JOINT = 6; + const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT; + if (neuronPlugin->_joints.size() != NUM_JOINTS) { + neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }); + } + + assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); + + // copy the data + memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); + + } else { + static bool ONCE = false; + if (!ONCE) { + qCCritical(inputplugins) << "NeuronPlugin: bad frame version number, expected 1.0"; + ONCE = true; + } } - - assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float))); - - // copy the data - memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS); } // NOTE: must be thread-safe static void CommandDataReceivedCallback(void* context, SOCKET_REF sender, CommandPack* pack, void* data) { + DATA_VER version; + version._VersionMask = pack->DataVersion; + if (version.Major == 1 && version.Minor == 0) { + const char* str = "Unknown"; + switch (pack->CommandId) { + case Cmd_BoneSize: // Id can be used to request bone size from server or register avatar name command. + str = "BoneSize"; + break; + case Cmd_AvatarName: // Id can be used to request avatar name from server or register avatar name command. + str = "AvatarName"; + break; + case Cmd_FaceDirection: // Id used to request face direction from server + str = "FaceDirection"; + break; + case Cmd_DataFrequency: // Id can be used to request data frequency from server or register data frequency command. + str = "DataFrequency"; + break; + case Cmd_BvhInheritanceTxt: // Id can be used to request bvh header txt from server or register bvh header txt command. + str = "BvhInheritanceTxt"; + break; + case Cmd_AvatarCount: // Id can be used to request avatar count from server or register avatar count command. + str = "AvatarCount"; + break; + case Cmd_CombinationMode: // Id can be used to request combination mode from server or register combination mode command. + str = "CombinationMode"; + break; + case Cmd_RegisterEvent: // Id can be used to register event. + str = "RegisterEvent"; + break; + case Cmd_UnRegisterEvent: // Id can be used to unregister event. + str = "UnRegisterEvent"; + break; + } + qCDebug(inputplugins) << "NeuronPlugin: command data received CommandID = " << str; + } else { + static bool ONCE = false; + if (!ONCE) { + qCCritical(inputplugins) << "NeuronPlugin: bad command version number, expected 1.0"; + ONCE = true; + } + } } // NOTE: must be thread-safe @@ -130,6 +185,11 @@ static void SocketStatusChangedCallback(void* context, SOCKET_REF sender, Socket void NeuronPlugin::activate() { InputPlugin::activate(); + + // register with userInputMapper + auto userInputMapper = DependencyManager::get(); + userInputMapper->registerDevice(_inputDevice); + qCDebug(inputplugins) << "NeuronPlugin::activate"; // register c-style callbacks @@ -149,12 +209,18 @@ void NeuronPlugin::activate() { } void NeuronPlugin::deactivate() { - // TODO: qCDebug(inputplugins) << "NeuronPlugin::deactivate"; + // unregister from userInputMapper + if (_inputDevice->_deviceID != controller::Input::INVALID_DEVICE) { + auto userInputMapper = DependencyManager::get(); + userInputMapper->removeDevice(_inputDevice->_deviceID); + } + if (_socketRef) { BRCloseSocket(_socketRef); } + InputPlugin::deactivate(); } @@ -174,12 +240,14 @@ void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) { joints = _joints; } + /* DebugDraw::getInstance().addMyAvatarMarker("LEFT_FOOT", - eulerToQuat(joints[6].rot), + eulerToQuat(joints[6].euler), joints[6].pos / 100.0f, glm::vec4(1)); - - _inputDevice->update(deltaTime, jointsCaptured); + */ + _inputDevice->update(deltaTime, joints, _prevJoints); + _prevJoints = joints; } void NeuronPlugin::saveSettings() const { @@ -198,11 +266,86 @@ void NeuronPlugin::loadSettings() { // InputDevice // +static controller::StandardPoseChannel neuronJointIndexToPoseIndex(JointIndex i) { + // Currently they are the same. + // but that won't always be the case... + return (controller::StandardPoseChannel)i; +} + +static const char* neuronJointName(JointIndex i) { + switch (i) { + case HipsPosition: return "HipsPosition"; + case Hips: return "Hips"; + case RightUpLeg: return "RightUpLeg"; + case RightLeg: return "RightLeg"; + case RightFoot: return "RightFoot"; + case LeftUpLeg: return "LeftUpLeg"; + case LeftLeg: return "LeftLeg"; + case LeftFoot: return "LeftFoot"; + case Spine: return "Spine"; + case Spine1: return "Spine1"; + case Spine2: return "Spine2"; + case Spine3: return "Spine3"; + case Neck: return "Neck"; + case Head: return "Head"; + case RightShoulder: return "RightShoulder"; + case RightArm: return "RightArm"; + case RightForeArm: return "RightForeArm"; + case RightHand: return "RightHand"; + case RightHandThumb1: return "RightHandThumb1"; + case RightHandThumb2: return "RightHandThumb2"; + case RightHandThumb3: return "RightHandThumb3"; + case RightInHandIndex: return "RightInHandIndex"; + case RightHandIndex1: return "RightHandIndex1"; + case RightHandIndex2: return "RightHandIndex2"; + case RightHandIndex3: return "RightHandIndex3"; + case RightInHandMiddle: return "RightInHandMiddle"; + case RightHandMiddle1: return "RightHandMiddle1"; + case RightHandMiddle2: return "RightHandMiddle2"; + case RightHandMiddle3: return "RightHandMiddle3"; + case RightInHandRing: return "RightInHandRing"; + case RightHandRing1: return "RightHandRing1"; + case RightHandRing2: return "RightHandRing2"; + case RightHandRing3: return "RightHandRing3"; + case RightInHandPinky: return "RightInHandPinky"; + case RightHandPinky1: return "RightHandPinky1"; + case RightHandPinky2: return "RightHandPinky2"; + case RightHandPinky3: return "RightHandPinky3"; + case LeftShoulder: return "LeftShoulder"; + case LeftArm: return "LeftArm"; + case LeftForeArm: return "LeftForeArm"; + case LeftHand: return "LeftHand"; + case LeftHandThumb1: return "LeftHandThumb1"; + case LeftHandThumb2: return "LeftHandThumb2"; + case LeftHandThumb3: return "LeftHandThumb3"; + case LeftInHandIndex: return "LeftInHandIndex"; + case LeftHandIndex1: return "LeftHandIndex1"; + case LeftHandIndex2: return "LeftHandIndex2"; + case LeftHandIndex3: return "LeftHandIndex3"; + case LeftInHandMiddle: return "LeftInHandMiddle"; + case LeftHandMiddle1: return "LeftHandMiddle1"; + case LeftHandMiddle2: return "LeftHandMiddle2"; + case LeftHandMiddle3: return "LeftHandMiddle3"; + case LeftInHandRing: return "LeftInHandRing"; + case LeftHandRing1: return "LeftHandRing1"; + case LeftHandRing2: return "LeftHandRing2"; + case LeftHandRing3: return "LeftHandRing3"; + case LeftInHandPinky: return "LeftInHandPinky"; + case LeftHandPinky1: return "LeftHandPinky1"; + case LeftHandPinky2: return "LeftHandPinky2"; + case LeftHandPinky3: return "LeftHandPinky3"; + default: return "???"; + } +} + controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const { // TODO: - static const controller::Input::NamedVector availableInputs { - makePair(controller::LEFT_HAND, "LeftHand"), - makePair(controller::RIGHT_HAND, "RightHand") + static controller::Input::NamedVector availableInputs; + + if (availableInputs.size() == 0) { + for (int i = 0; i < JointIndex::Size; i++) { + availableInputs.push_back(makePair(neuronJointIndexToPoseIndex((JointIndex)i), neuronJointName((JointIndex)i))); + } }; return availableInputs; } @@ -212,8 +355,24 @@ QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const { return MAPPING_JSON; } -void NeuronPlugin::InputDevice::update(float deltaTime, bool jointsCaptured) { +void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector& joints, const std::vector& prevJoints) { + for (int i = 0; i < joints.size(); i++) { + int poseIndex = neuronJointIndexToPoseIndex((JointIndex)i); + glm::vec3 linearVel, angularVel; + glm::vec3 pos = (joints[i].pos * METERS_PER_CENTIMETER); + glm::quat rot = eulerToQuat(joints[i].euler); + if (i < prevJoints.size()) { + linearVel = (pos - (prevJoints[i].pos * METERS_PER_CENTIMETER)) / deltaTime; + // quat log imag part points along the axis of rotation, and it's length will be the half angle. + glm::quat d = glm::log(rot * glm::inverse(eulerToQuat(prevJoints[i].euler))); + angularVel = glm::vec3(d.x, d.y, d.z) / (0.5f * deltaTime); + } + _poseStateMap[poseIndex] = controller::Pose(pos, rot, linearVel, angularVel); + if (glm::length(angularVel) > 0.5f) { + qCDebug(inputplugins) << "Movement in joint" << i << neuronJointName((JointIndex)i); + } + } } void NeuronPlugin::InputDevice::focusOutEvent() { diff --git a/plugins/hifiNeuron/src/NeuronPlugin.h b/plugins/hifiNeuron/src/NeuronPlugin.h index 5f67502e04..f787838ce2 100644 --- a/plugins/hifiNeuron/src/NeuronPlugin.h +++ b/plugins/hifiNeuron/src/NeuronPlugin.h @@ -41,15 +41,25 @@ public: virtual void loadSettings() override; protected: + + struct NeuronJoint { + glm::vec3 pos; + glm::vec3 euler; + }; + class InputDevice : public controller::InputDevice { public: + friend class NeuronPlugin; + InputDevice() : controller::InputDevice("Neuron") {} // Device functions virtual controller::Input::NamedVector getAvailableInputs() const override; virtual QString getDefaultMappingConfig() const override; - virtual void update(float deltaTime, bool jointsCaptured) override; + virtual void update(float deltaTime, bool jointsCaptured) override {}; virtual void focusOutEvent() override; + + void update(float deltaTime, const std::vector& joints, const std::vector& prevJoints); }; std::shared_ptr _inputDevice { std::make_shared() }; @@ -61,13 +71,10 @@ protected: int _serverPort; void* _socketRef; - struct NeuronJoint { - glm::vec3 pos; - glm::vec3 rot; - }; - std::vector _joints; - std::mutex _jointsMutex; + std::mutex _jointsMutex; // used to guard access to _joints + + std::vector _prevJoints; }; #endif // hifi_NeuronPlugin_h