Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-08 13:14:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Copy Neuron joints into controller poses

Browse source 
This makes the accessible for controller mapping and to JavaScript.

Added 'neuronAvatar.js' as an example of reading joints from the neuron and setting them
on the avatar.  NOTE: the rotations are currently in the wrong coordinate frame.
This commit is contained in:
Anthony J. Thibault 2015-12-22 17:21:33 -08:00
parent 0459479c2b
commit 6afe3bae5e
5 changed files with 400 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

141
examples/controllers/neuron/neuronAvatar.js Normal file
View file

@ -0,0 +1,141 @@
// maps controller joint names to avatar joint names
var JOINT_NAME_MAP = {
HipsPosition: "",
Hips: "Hips",
RightUpLeg: "RightUpLeg",
RightLeg: "RightLeg",
RightFoot: "RightFoot",
LeftUpLeg: "LeftUpLeg",
LeftLeg: "LeftLeg",
LeftFoot: "LeftFoot",
Spine: "Spine",
Spine1: "Spine1",
Spine2: "Spine2",
Spine3: "Spine3",
Neck: "Neck",
Head: "Head",
RightShoulder: "RightShoulder",
RightArm: "RightArm",
RightForeArm: "RightForeArm",
RightHand: "RightHand",
RightHandThumb1: "RightHandThumb2",
RightHandThumb2: "RightHandThumb3",
RightHandThumb3: "RightHandThumb4",
RightInHandIndex: "RightHandIndex1",
RightHandIndex1: "RightHandIndex2",
RightHandIndex2: "RightHandIndex3",
RightHandIndex3: "RightHandIndex4",
RightInHandMiddle: "RightHandMiddle1",
RightHandMiddle1: "RightHandMiddle2",
RightHandMiddle2: "RightHandMiddle3",
RightHandMiddle3: "RightHandMiddle4",
RightInHandRing: "RightHandRing1",
RightHandRing1: "RightHandRing2",
RightHandRing2: "RightHandRing3",
RightHandRing3: "RightHandRing4",
RightInHandPinky: "RightHandPinky1",
RightHandPinky1: "RightHandPinky2",
RightHandPinky2: "RightHandPinky3",
RightHandPinky3: "RightHandPinky4",
LeftShoulder: "LeftShoulder",
LeftArm: "LeftArm",
LeftForeArm: "LeftForeArm",
LeftHand: "LeftHand",
LeftHandThumb1: "LeftHandThumb2",
LeftHandThumb2: "LeftHandThumb3",
LeftHandThumb3: "LeftHandThumb4",
LeftInHandIndex: "LeftHandIndex1",
LeftHandIndex1: "LeftHandIndex2",
LeftHandIndex2: "LeftHandIndex3",
LeftHandIndex3: "LeftHandIndex4",
LeftInHandMiddle: "LeftHandMiddle1",
LeftHandMiddle1: "LeftHandMiddle2",
LeftHandMiddle2: "LeftHandMiddle3",
LeftHandMiddle3: "LeftHandMiddle4",
LeftInHandRing: "LeftHandRing1",
LeftHandRing1: "LeftHandRing2",
LeftHandRing2: "LeftHandRing3",
LeftHandRing3: "LeftHandRing4",
LeftInHandPinky: "LeftHandPinky1",
LeftHandPinky1: "LeftHandPinky2",
LeftHandPinky2: "LeftHandPinky3",
LeftHandPinky3: "LeftHandPinky4"
};
function dumpHardwareMapping() {
Object.keys(Controller.Hardware).forEach(function (deviceName) {
Object.keys(Controller.Hardware[deviceName]).forEach(function (input) {
print("Controller.Hardware." + deviceName + "." + input + ":" + Controller.Hardware[deviceName][input]);
});
});
}
// ctor
function NeuronAvatar() {
var self = this;
Script.scriptEnding.connect(function () {
self.shutdown();
});
Controller.hardwareChanged.connect(function () {
self.hardwareChanged();
});
if (Controller.Hardware.Neuron) {
this.activate();
} else {
this.deactivate();
}
}
NeuronAvatar.prototype.shutdown = function () {
this.deactivate();
};
NeuronAvatar.prototype.hardwareChanged = function () {
if (Controller.Hardware.Neuron) {
this.activate();
} else {
this.deactivate();
}
};
NeuronAvatar.prototype.activate = function () {
if (!this._active) {
Script.update.connect(updateCallback);
}
this._active = true;
};
NeuronAvatar.prototype.deactivate = function () {
if (this._active) {
var self = this;
Script.update.disconnect(updateCallback);
}
this._active = false;
MyAvatar.clearJointsData();
};
NeuronAvatar.prototype.update = function (deltaTime) {
var keys = Object.keys(JOINT_NAME_MAP);
var i, l = keys.length;
for (i = 0; i < l; i++) {
var channel = Controller.Hardware.Neuron[keys[i]];
if (channel) {
var pose = Controller.getPoseValue(channel);
var j = MyAvatar.getJointIndex(JOINT_NAME_MAP[keys[i]]);
var defaultRot = MyAvatar.getDefaultJointRotation(j);
if (keys[i] == "Hips") {
MyAvatar.setJointRotation(j, Quat.multiply(pose.rotation, defaultRot));
} else {
MyAvatar.setJointRotation(j, defaultRot);
}
}
}
};
var neuronAvatar = new NeuronAvatar();
function updateCallback(deltaTime) {
neuronAvatar.update(deltaTime);
}

6
libraries/animation/src/Rig.cpp
View file

@ -289,8 +289,10 @@ void Rig::clearJointState(int index) {
void Rig::clearJointStates() {
_internalPoseSet._overrideFlags.clear();
_internalPoseSet._overrideFlags.resize(_animSkeleton->getNumJoints());
_internalPoseSet._overridePoses = _animSkeleton->getRelativeDefaultPoses();
if (_animSkeleton) {
_internalPoseSet._overrideFlags.resize(_animSkeleton->getNumJoints());
_internalPoseSet._overridePoses = _animSkeleton->getRelativeDefaultPoses();
}
}
void Rig::clearJointAnimationPriority(int index) {

61
libraries/controllers/src/controllers/StandardControls.h
View file

@ -88,9 +88,66 @@ namespace controller {
// No correlation to SDL
enum StandardPoseChannel {
LEFT_HAND = 0,
RIGHT_HAND,
HIPS_ROOT = 0,
HIPS,
RIGHT_UP_LEG,
RIGHT_LEG,
RIGHT_FOOT,
LEFT_UP_LEG,
LEFT_LEG,
LEFT_FOOT,
SPINE,
SPINE1,
SPINE2,
SPINE3,
NECK,
HEAD,
RIGHT_SHOULDER,
RIGHT_ARM,
RIGHT_FORE_ARM,
RIGHT_HAND,
RIGHT_HAND_THUMB1,
RIGHT_HAND_THUMB2,
RIGHT_HAND_THUMB3,
RIGHT_IN_HAND_INDEX,
RIGHT_HAND_INDEX1,
RIGHT_HAND_INDEX2,
RIGHT_HAND_INDEX3,
RIGHT_IN_HAND_MIDDLE,
RIGHT_HAND_MIDDLE1,
RIGHT_HAND_MIDDLE2,
RIGHT_HAND_MIDDLE3,
RIGHT_IN_HANDRING,
RIGHT_HAND_RING1,
RIGHT_HAND_RING2,
RIGHT_HAND_RING3,
RIGHT_IN_HAND_PINKY,
RIGHT_HAND_PINKY1,
RIGHT_HAND_PINKY2,
RIGHT_HAND_PINKY3,
LEFT_SHOULDER,
LEFT_ARM,
LEFT_FORE_ARM,
LEFT_HAND,
LEFT_HAND_THUMB1,
LEFT_HAND_THUMB2,
LEFT_HAND_THUMB3,
LEFT_IN_HAND_INDEX,
LEFT_HAND_INDEX1,
LEFT_HAND_INDEX2,
LEFT_HAND_INDEX3,
LEFT_IN_HAND_MIDDLE,
LEFT_HAND_MIDDLE1,
LEFT_HAND_MIDDLE2,
LEFT_HAND_MIDDLE3,
LEFT_IN_HAND_RING,
LEFT_HAND_RING1,
LEFT_HAND_RING2,
LEFT_HAND_RING3,
LEFT_IN_HAND_PINKY,
LEFT_HAND_PINKY1,
LEFT_HAND_PINKY2,
LEFT_HAND_PINKY3,
NUM_STANDARD_POSES
};

205
plugins/hifiNeuron/src/NeuronPlugin.cpp
View file

@ -1,5 +1,5 @@
//
// NeuronPlugin.h
// NeuronPlugin.cpp
// input-plugins/src/input-plugins
//
// Created by Anthony Thibault on 12/18/2015.
@ -11,6 +11,7 @@
#include "NeuronPlugin.h"
#include <controllers/UserInputMapper.h>
#include <QLoggingCategory>
#include <PathUtils.h>
#include <DebugDraw.h>
@ -27,6 +28,7 @@ Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins")
const QString NeuronPlugin::NAME = "Neuron";
const QString NeuronPlugin::NEURON_ID_STRING = "Perception Neuron";
// This matches controller::StandardPoseChannel
enum JointIndex {
HipsPosition = 0,
Hips,
@ -87,7 +89,8 @@ enum JointIndex {
LeftInHandPinky,
LeftHandPinky1,
LeftHandPinky2,
LeftHandPinky3
LeftHandPinky3,
Size
};
bool NeuronPlugin::isSupported() const {
@ -98,29 +101,81 @@ bool NeuronPlugin::isSupported() const {
// NOTE: must be thread-safe
void FrameDataReceivedCallback(void* context, SOCKET_REF sender, BvhDataHeaderEx* header, float* data) {
qCDebug(inputplugins) << "NeuronPlugin: received frame data, DataCount = " << header->DataCount;
auto neuronPlugin = reinterpret_cast<NeuronPlugin*>(context);
std::lock_guard<std::mutex> guard(neuronPlugin->_jointsMutex);
// Data is 6 floats: 3 position values, 3 rotation euler angles (degrees)
// version 1.0
if (header->DataVersion.Major == 1 && header->DataVersion.Minor == 0) {
// resize vector if necessary
const size_t NUM_FLOATS_PER_JOINT = 6;
const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT;
if (neuronPlugin->_joints.size() != NUM_JOINTS) {
neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } });
std::lock_guard<std::mutex> guard(neuronPlugin->_jointsMutex);
// Data is 6 floats: 3 position values, 3 rotation euler angles (degrees)
// resize vector if necessary
const size_t NUM_FLOATS_PER_JOINT = 6;
const size_t NUM_JOINTS = header->DataCount / NUM_FLOATS_PER_JOINT;
if (neuronPlugin->_joints.size() != NUM_JOINTS) {
neuronPlugin->_joints.resize(NUM_JOINTS, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } });
}
assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float)));
// copy the data
memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS);
} else {
static bool ONCE = false;
if (!ONCE) {
qCCritical(inputplugins) << "NeuronPlugin: bad frame version number, expected 1.0";
ONCE = true;
}
}
assert(sizeof(NeuronPlugin::NeuronJoint) == (NUM_FLOATS_PER_JOINT * sizeof(float)));
// copy the data
memcpy(&(neuronPlugin->_joints[0]), data, sizeof(NeuronPlugin::NeuronJoint) * NUM_JOINTS);
}
// NOTE: must be thread-safe
static void CommandDataReceivedCallback(void* context, SOCKET_REF sender, CommandPack* pack, void* data) {
DATA_VER version;
version._VersionMask = pack->DataVersion;
if (version.Major == 1 && version.Minor == 0) {
const char* str = "Unknown";
switch (pack->CommandId) {
case Cmd_BoneSize: // Id can be used to request bone size from server or register avatar name command.
str = "BoneSize";
break;
case Cmd_AvatarName: // Id can be used to request avatar name from server or register avatar name command.
str = "AvatarName";
break;
case Cmd_FaceDirection: // Id used to request face direction from server
str = "FaceDirection";
break;
case Cmd_DataFrequency: // Id can be used to request data frequency from server or register data frequency command.
str = "DataFrequency";
break;
case Cmd_BvhInheritanceTxt: // Id can be used to request bvh header txt from server or register bvh header txt command.
str = "BvhInheritanceTxt";
break;
case Cmd_AvatarCount: // Id can be used to request avatar count from server or register avatar count command.
str = "AvatarCount";
break;
case Cmd_CombinationMode: // Id can be used to request combination mode from server or register combination mode command.
str = "CombinationMode";
break;
case Cmd_RegisterEvent: // Id can be used to register event.
str = "RegisterEvent";
break;
case Cmd_UnRegisterEvent: // Id can be used to unregister event.
str = "UnRegisterEvent";
break;
}
qCDebug(inputplugins) << "NeuronPlugin: command data received CommandID = " << str;
} else {
static bool ONCE = false;
if (!ONCE) {
qCCritical(inputplugins) << "NeuronPlugin: bad command version number, expected 1.0";
ONCE = true;
}
}
}
// NOTE: must be thread-safe
@ -130,6 +185,11 @@ static void SocketStatusChangedCallback(void* context, SOCKET_REF sender, Socket
void NeuronPlugin::activate() {
InputPlugin::activate();
// register with userInputMapper
auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
userInputMapper->registerDevice(_inputDevice);
qCDebug(inputplugins) << "NeuronPlugin::activate";
// register c-style callbacks
@ -149,12 +209,18 @@ void NeuronPlugin::activate() {
}
void NeuronPlugin::deactivate() {
// TODO:
qCDebug(inputplugins) << "NeuronPlugin::deactivate";
// unregister from userInputMapper
if (_inputDevice->_deviceID != controller::Input::INVALID_DEVICE) {
auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
userInputMapper->removeDevice(_inputDevice->_deviceID);
}
if (_socketRef) {
BRCloseSocket(_socketRef);
}
InputPlugin::deactivate();
}
@ -174,12 +240,14 @@ void NeuronPlugin::pluginUpdate(float deltaTime, bool jointsCaptured) {
joints = _joints;
}
/*
DebugDraw::getInstance().addMyAvatarMarker("LEFT_FOOT",
eulerToQuat(joints[6].rot),
eulerToQuat(joints[6].euler),
joints[6].pos / 100.0f,
glm::vec4(1));
_inputDevice->update(deltaTime, jointsCaptured);
*/
_inputDevice->update(deltaTime, joints, _prevJoints);
_prevJoints = joints;
}
void NeuronPlugin::saveSettings() const {
@ -198,11 +266,86 @@ void NeuronPlugin::loadSettings() {
// InputDevice
//
static controller::StandardPoseChannel neuronJointIndexToPoseIndex(JointIndex i) {
// Currently they are the same.
// but that won't always be the case...
return (controller::StandardPoseChannel)i;
}
static const char* neuronJointName(JointIndex i) {
switch (i) {
case HipsPosition: return "HipsPosition";
case Hips: return "Hips";
case RightUpLeg: return "RightUpLeg";
case RightLeg: return "RightLeg";
case RightFoot: return "RightFoot";
case LeftUpLeg: return "LeftUpLeg";
case LeftLeg: return "LeftLeg";
case LeftFoot: return "LeftFoot";
case Spine: return "Spine";
case Spine1: return "Spine1";
case Spine2: return "Spine2";
case Spine3: return "Spine3";
case Neck: return "Neck";
case Head: return "Head";
case RightShoulder: return "RightShoulder";
case RightArm: return "RightArm";
case RightForeArm: return "RightForeArm";
case RightHand: return "RightHand";
case RightHandThumb1: return "RightHandThumb1";
case RightHandThumb2: return "RightHandThumb2";
case RightHandThumb3: return "RightHandThumb3";
case RightInHandIndex: return "RightInHandIndex";
case RightHandIndex1: return "RightHandIndex1";
case RightHandIndex2: return "RightHandIndex2";
case RightHandIndex3: return "RightHandIndex3";
case RightInHandMiddle: return "RightInHandMiddle";
case RightHandMiddle1: return "RightHandMiddle1";
case RightHandMiddle2: return "RightHandMiddle2";
case RightHandMiddle3: return "RightHandMiddle3";
case RightInHandRing: return "RightInHandRing";
case RightHandRing1: return "RightHandRing1";
case RightHandRing2: return "RightHandRing2";
case RightHandRing3: return "RightHandRing3";
case RightInHandPinky: return "RightInHandPinky";
case RightHandPinky1: return "RightHandPinky1";
case RightHandPinky2: return "RightHandPinky2";
case RightHandPinky3: return "RightHandPinky3";
case LeftShoulder: return "LeftShoulder";
case LeftArm: return "LeftArm";
case LeftForeArm: return "LeftForeArm";
case LeftHand: return "LeftHand";
case LeftHandThumb1: return "LeftHandThumb1";
case LeftHandThumb2: return "LeftHandThumb2";
case LeftHandThumb3: return "LeftHandThumb3";
case LeftInHandIndex: return "LeftInHandIndex";
case LeftHandIndex1: return "LeftHandIndex1";
case LeftHandIndex2: return "LeftHandIndex2";
case LeftHandIndex3: return "LeftHandIndex3";
case LeftInHandMiddle: return "LeftInHandMiddle";
case LeftHandMiddle1: return "LeftHandMiddle1";
case LeftHandMiddle2: return "LeftHandMiddle2";
case LeftHandMiddle3: return "LeftHandMiddle3";
case LeftInHandRing: return "LeftInHandRing";
case LeftHandRing1: return "LeftHandRing1";
case LeftHandRing2: return "LeftHandRing2";
case LeftHandRing3: return "LeftHandRing3";
case LeftInHandPinky: return "LeftInHandPinky";
case LeftHandPinky1: return "LeftHandPinky1";
case LeftHandPinky2: return "LeftHandPinky2";
case LeftHandPinky3: return "LeftHandPinky3";
default: return "???";
}
}
controller::Input::NamedVector NeuronPlugin::InputDevice::getAvailableInputs() const {
// TODO:
static const controller::Input::NamedVector availableInputs {
makePair(controller::LEFT_HAND, "LeftHand"),
makePair(controller::RIGHT_HAND, "RightHand")
static controller::Input::NamedVector availableInputs;
if (availableInputs.size() == 0) {
for (int i = 0; i < JointIndex::Size; i++) {
availableInputs.push_back(makePair(neuronJointIndexToPoseIndex((JointIndex)i), neuronJointName((JointIndex)i)));
}
};
return availableInputs;
}
@ -212,8 +355,24 @@ QString NeuronPlugin::InputDevice::getDefaultMappingConfig() const {
return MAPPING_JSON;
}
void NeuronPlugin::InputDevice::update(float deltaTime, bool jointsCaptured) {
void NeuronPlugin::InputDevice::update(float deltaTime, const std::vector<NeuronPlugin::NeuronJoint>& joints, const std::vector<NeuronPlugin::NeuronJoint>& prevJoints) {
for (int i = 0; i < joints.size(); i++) {
int poseIndex = neuronJointIndexToPoseIndex((JointIndex)i);
glm::vec3 linearVel, angularVel;
glm::vec3 pos = (joints[i].pos * METERS_PER_CENTIMETER);
glm::quat rot = eulerToQuat(joints[i].euler);
if (i < prevJoints.size()) {
linearVel = (pos - (prevJoints[i].pos * METERS_PER_CENTIMETER)) / deltaTime;
// quat log imag part points along the axis of rotation, and it's length will be the half angle.
glm::quat d = glm::log(rot * glm::inverse(eulerToQuat(prevJoints[i].euler)));
angularVel = glm::vec3(d.x, d.y, d.z) / (0.5f * deltaTime);
}
_poseStateMap[poseIndex] = controller::Pose(pos, rot, linearVel, angularVel);
if (glm::length(angularVel) > 0.5f) {
qCDebug(inputplugins) << "Movement in joint" << i << neuronJointName((JointIndex)i);
}
}
}
void NeuronPlugin::InputDevice::focusOutEvent() {

21
plugins/hifiNeuron/src/NeuronPlugin.h
View file

@ -41,15 +41,25 @@ public:
virtual void loadSettings() override;
protected:
struct NeuronJoint {
glm::vec3 pos;
glm::vec3 euler;
};
class InputDevice : public controller::InputDevice {
public:
friend class NeuronPlugin;
InputDevice() : controller::InputDevice("Neuron") {}
// Device functions
virtual controller::Input::NamedVector getAvailableInputs() const override;
virtual QString getDefaultMappingConfig() const override;
virtual void update(float deltaTime, bool jointsCaptured) override;
virtual void update(float deltaTime, bool jointsCaptured) override {};
virtual void focusOutEvent() override;
void update(float deltaTime, const std::vector<NeuronPlugin::NeuronJoint>& joints, const std::vector<NeuronPlugin::NeuronJoint>& prevJoints);
};
std::shared_ptr<InputDevice> _inputDevice { std::make_shared<InputDevice>() };
@ -61,13 +71,10 @@ protected:
int _serverPort;
void* _socketRef;
struct NeuronJoint {
glm::vec3 pos;
glm::vec3 rot;
};
std::vector<NeuronJoint> _joints;
std::mutex _jointsMutex;
std::mutex _jointsMutex; // used to guard access to _joints
std::vector<NeuronJoint> _prevJoints;
};
#endif // hifi_NeuronPlugin_h