From 3f26e1b63ad18ab228b2138e6ab9f4d3f068909f Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Sat, 13 Dec 2014 23:04:19 +0100 Subject: [PATCH 01/11] RealSense Initial --- cmake/modules/FindRSSDK.cmake | 33 ++++ interface/CMakeLists.txt | 2 +- interface/src/Application.cpp | 2 + interface/src/Menu.cpp | 10 ++ interface/src/Menu.h | 2 + interface/src/devices/RealSense.cpp | 233 ++++++++++++++++++++++++++++ interface/src/devices/RealSense.h | 63 ++++++++ 7 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 cmake/modules/FindRSSDK.cmake create mode 100644 interface/src/devices/RealSense.cpp create mode 100644 interface/src/devices/RealSense.h diff --git a/cmake/modules/FindRSSDK.cmake b/cmake/modules/FindRSSDK.cmake new file mode 100644 index 0000000000..c31b0efcd9 --- /dev/null +++ b/cmake/modules/FindRSSDK.cmake @@ -0,0 +1,33 @@ +# Try to find the RSSDK library +# +# You must provide a RSSDK_ROOT_DIR which contains lib and include directories +# +# Once done this will define +# +# RSSDK_FOUND - system found RSSDK +# RSSDK_INCLUDE_DIRS - the RSSDK include directory +# RSSDK_LIBRARIES - Link this to use RSSDK +# +# Created on 12/7/2014 by Thijs Wenker +# Copyright (c) 2014 High Fidelity +# + +include("${MACRO_DIR}/HifiLibrarySearchHints.cmake") +hifi_library_search_hints("rssdk") + +find_path(RSSDK_INCLUDE_DIRS pxcbase.h PATH_SUFFIXES include HINTS ${RSSDK_SEARCH_DIRS}) + +if (WIN32) + find_library(RSSDK_LIBRARY_DEBUG libpxc_d PATH_SUFFIXES lib/Win32 HINTS ${RSSDK_SEARCH_DIRS}) + find_library(RSSDK_LIBRARY_RELEASE libpxc PATH_SUFFIXES lib/Win32 HINTS ${RSSDK_SEARCH_DIRS}) +endif () + +include(SelectLibraryConfigurations) +select_library_configurations(RSSDK) + +set(RSSDK_LIBRARIES "${RSSDK_LIBRARY}") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(RSSDK DEFAULT_MSG RSSDK_INCLUDE_DIRS RSSDK_LIBRARIES) + +mark_as_advanced(RSSDK_INCLUDE_DIRS RSSDK_LIBRARIES RSSDK_SEARCH_DIRS) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 38dd02c655..bc25778940 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -2,7 +2,7 @@ set(TARGET_NAME interface) project(${TARGET_NAME}) # set a default root dir for each of our optional externals if it was not passed -set(OPTIONAL_EXTERNALS "Faceshift" "LibOVR" "PrioVR" "Sixense" "Visage" "LeapMotion" "RtMidi" "Qxmpp" "SDL2" "Gverb") +set(OPTIONAL_EXTERNALS "Faceshift" "LibOVR" "PrioVR" "Sixense" "Visage" "LeapMotion" "RtMidi" "Qxmpp" "SDL2" "Gverb" "RSSDK") foreach(EXTERNAL ${OPTIONAL_EXTERNALS}) string(TOUPPER ${EXTERNAL} ${EXTERNAL}_UPPERCASE) if (NOT ${${EXTERNAL}_UPPERCASE}_ROOT_DIR) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 3824185f52..08e3a78653 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -76,6 +76,7 @@ #include "Util.h" #include "devices/Leapmotion.h" +#include "devices/RealSense.h" #include "devices/MIDIManager.h" #include "devices/OculusManager.h" #include "devices/TV3DManager.h" @@ -1994,6 +1995,7 @@ void Application::init() { _visage.init(); Leapmotion::init(); + RealSense::init(); // fire off an immediate domain-server check in now that settings are loaded NodeList::getInstance()->sendDomainServerCheckIn(); diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index ae12b5ff26..37e4d96ed6 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -49,6 +49,7 @@ #include "ui/ModelsBrowser.h" #include "ui/LoginDialog.h" #include "ui/NodeBounds.h" +#include "devices/RealSense.h" #include "devices/OculusManager.h" @@ -493,6 +494,11 @@ Menu::Menu() : QMenu* leapOptionsMenu = handOptionsMenu->addMenu("Leap Motion"); addCheckableActionToQMenuAndActionHash(leapOptionsMenu, MenuOption::LeapMotionOnHMD, 0, false); +#ifdef HAVE_RSSDK + QMenu* realSenseOptionsMenu = handOptionsMenu->addMenu("RealSense"); + addActionToQMenuAndActionHash(realSenseOptionsMenu, MenuOption::LoadRSSDKFile, 0, this, SLOT(loadRSSDKFile())); +#endif + QMenu* networkMenu = developerMenu->addMenu("Network"); addCheckableActionToQMenuAndActionHash(networkMenu, MenuOption::DisableNackPackets, 0, false); addCheckableActionToQMenuAndActionHash(networkMenu, @@ -1346,6 +1352,10 @@ void Menu::showChat() { } } +void Menu::loadRSSDKFile() { + RealSense::getInstance()->loadRSSDKFile(); +} + void Menu::toggleChat() { #ifdef HAVE_QXMPP _chatAction->setEnabled(XmppClient::getInstance().getXMPPClient().isConnected()); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 138828d3e8..4ed5be8057 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -236,6 +236,7 @@ private slots: void displayAddressNotFoundMessage(); void muteEnvironment(); void changeVSync(); + void loadRSSDKFile(); private: static Menu* _instance; @@ -414,6 +415,7 @@ namespace MenuOption { const QString LeapMotionOnHMD = "Leap Motion on HMD"; const QString LoadScript = "Open and Run Script File..."; const QString LoadScriptURL = "Open and Run Script from URL..."; + const QString LoadRSSDKFile = "Load .rssdk file"; const QString LodTools = "LOD Tools"; const QString Login = "Login"; const QString Log = "Log"; diff --git a/interface/src/devices/RealSense.cpp b/interface/src/devices/RealSense.cpp new file mode 100644 index 0000000000..08492d8377 --- /dev/null +++ b/interface/src/devices/RealSense.cpp @@ -0,0 +1,233 @@ +// +// RealSense.cpp +// interface/src/devices +// +// Created by Thijs Wenker on 12/10/14 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "RealSense.h" +#include "Menu.h" +#include "SharedUtil.h" + +const int PALMROOT_NUM_JOINTS = 2; +const int FINGER_NUM_JOINTS = 4; + +const DeviceTracker::Name RealSense::NAME = "RealSense"; + +// find the index of a joint from +// the side: true = right +// the finger & the bone: +// finger in [0..4] : bone in [0..3] a finger phalange +// [-1] up the hand branch : bone in [0..2] <=> [ hand, forearm, arm] +MotionTracker::Index evalRealSenseJointIndex(bool isRightSide, int finger, int bone) { + + MotionTracker::Index offset = 1 // start after root + + (int(isRightSide) * PXCHandData::NUMBER_OF_JOINTS) // then offset for side + + PALMROOT_NUM_JOINTS; // then add the arm/forearm/hand chain + if (finger >= 0) { + // from there go down in the correct finger and bone + return offset + (finger * FINGER_NUM_JOINTS) + bone; + } else { + // or go back up for the correct root bone + return offset - 1 - bone; + } +} + +// static +void RealSense::init() { + DeviceTracker* device = DeviceTracker::getDevice(NAME); + if (!device) { + // create a new RealSense and register it + RealSense* realSense = new RealSense(); + DeviceTracker::registerDevice(NAME, realSense); + } +} + +// static +RealSense* RealSense::getInstance() { + DeviceTracker* device = DeviceTracker::getDevice(NAME); + if (!device) { + // create a new RealSense and register it + RealSense* realSense = new RealSense(); + DeviceTracker::registerDevice(NAME, realSense); + } + return dynamic_cast< RealSense* > (device); +} + +RealSense::RealSense() : + MotionTracker(), + _active(false), + _handData(NULL) +{ + _session = PXCSession_Create(); + initSession(false, NULL); + + // Create the RealSense joint hierarchy + std::vector< Semantic > sides; + sides.push_back("joint_L_"); + sides.push_back("joint_R_"); + + std::vector< Semantic > rootBones; + rootBones.push_back("wrist"); + rootBones.push_back("hand"); + + std::vector< Semantic > fingers; + fingers.push_back("thumb"); + fingers.push_back("index"); + fingers.push_back("middle"); + fingers.push_back("ring"); + fingers.push_back("pinky"); + + std::vector< Semantic > fingerBones; + fingerBones.push_back("1"); + fingerBones.push_back("2"); + fingerBones.push_back("3"); + fingerBones.push_back("4"); + + std::vector< Index > palms; + for (unsigned int s = 0; s < sides.size(); s++) { + Index rootJoint = 0; + for (unsigned int rb = 0; rb < rootBones.size(); rb++) { + rootJoint = addJoint(sides[s] + rootBones[rb], rootJoint); + } + + // capture the hand index for debug + palms.push_back(rootJoint); + + for (unsigned int f = 0; f < fingers.size(); f++) { + for (unsigned int b = 0; b < fingerBones.size(); b++) { + rootJoint = addJoint(sides[s] + fingers[f] + fingerBones[b], rootJoint); + } + } + } +} + +RealSense::~RealSense() { + _manager->Release(); +} + +void RealSense::initSession(bool playback, QString filename) { + _active = false; + _properlyInitialized = false; + if (_handData != NULL) { + _handData->Release(); + _handController->Release(); + _session->Release(); + } + _manager = _session->CreateSenseManager(); + if (playback) { + _manager->QueryCaptureManager()->SetFileName(filename.toStdWString().c_str(), false); + } + _manager->QueryCaptureManager()->SetRealtime(!playback); + _manager->EnableHand(0); + _handController = _manager->QueryHand(); + + if (_manager->Init() == PXC_STATUS_NO_ERROR) { + _handData = _handController->CreateOutput(); + + PXCCapture::Device *device = _manager->QueryCaptureManager()->QueryDevice(); + PXCCapture::DeviceInfo dinfo; + _manager->QueryCaptureManager()->QueryDevice()->QueryDeviceInfo(&dinfo); + if (dinfo.model == PXCCapture::DEVICE_MODEL_IVCAM) + { + device->SetDepthConfidenceThreshold(1); + device->SetMirrorMode(PXCCapture::Device::MIRROR_MODE_DISABLED); + device->SetIVCAMFilterOption(6); + } + _properlyInitialized = true; + } +} + +#ifdef HAVE_RSSDK +glm::quat quatFromPXCPoint4DF32(const PXCPoint4DF32& basis) { + return glm::quat(basis.w, basis.x, basis.y, basis.z); +} + +glm::vec3 vec3FromPXCPoint3DF32(const PXCPoint3DF32& vec) { + return glm::vec3(vec.x, vec.y, vec.z); +} +#endif + +void RealSense::update() { +#ifdef HAVE_RSSDK + bool wasActive = _active; + _active = _manager->IsConnected() && _properlyInitialized; + if (_active || wasActive) { + // Go through all the joints and increment their counter since last update. + // Increment all counters once after controller first becomes inactive so that each joint reports itself as inactive. + // TODO C++11 for (auto jointIt = _jointsArray.begin(); jointIt != _jointsArray.end(); jointIt++) { + for (JointTracker::Vector::iterator jointIt = _jointsArray.begin(); jointIt != _jointsArray.end(); jointIt++) { + (*jointIt).tickNewFrame(); + } + } + + if (!_active) { + return; + } + + pxcStatus sts = _manager->AcquireFrame(true); + _handData->Update(); + PXCHandData::JointData nodes[2][PXCHandData::NUMBER_OF_JOINTS] = {}; + PXCHandData::ExtremityData extremitiesPointsNodes[2][PXCHandData::NUMBER_OF_EXTREMITIES] = {}; + for (pxcI32 i = 0; i < _handData->QueryNumberOfHands(); i++) { + PXCHandData::IHand* handData; + if (_handData->QueryHandData(PXCHandData::ACCESS_ORDER_BY_TIME, i, handData) == PXC_STATUS_NO_ERROR) { + int rightSide = handData->QueryBodySide() == PXCHandData::BODY_SIDE_RIGHT; + PXCHandData::JointData jointData; + JointTracker* parentJointTracker = _jointsArray.data(); + //Iterate Joints + int rootBranchIndex = -1; + JointTracker* palmJoint = NULL; + for (int j = 0; j < PXCHandData::NUMBER_OF_JOINTS; j++) { + handData->QueryTrackedJoint((PXCHandData::JointType)j, jointData); + nodes[i][j] = jointData; + if (j == PXCHandData::JOINT_WRIST) { + JointTracker* wrist = editJointTracker(evalRealSenseJointIndex(rightSide, rootBranchIndex, 1)); // 1 is the index of the wrist joint + wrist->editAbsFrame().setTranslation(vec3FromPXCPoint3DF32(jointData.positionWorld)); + wrist->editAbsFrame().setRotation(quatFromPXCPoint4DF32(jointData.globalOrientation)); + wrist->updateLocFromAbsTransform(parentJointTracker); + wrist->activeFrame(); + parentJointTracker = wrist; + continue; + } else if (j == PXCHandData::JOINT_CENTER) { + palmJoint = editJointTracker(evalRealSenseJointIndex(rightSide, rootBranchIndex, 0)); // 0 is the index of the palm joint + palmJoint->editAbsFrame().setTranslation(vec3FromPXCPoint3DF32(jointData.positionWorld)); + palmJoint->editAbsFrame().setRotation(quatFromPXCPoint4DF32(jointData.globalOrientation)); + palmJoint->updateLocFromAbsTransform(parentJointTracker); + palmJoint->activeFrame(); + parentJointTracker = palmJoint; + continue; + } + int finger_index = j - PALMROOT_NUM_JOINTS; + int finger = finger_index / FINGER_NUM_JOINTS; + int finger_bone = finger_index % FINGER_NUM_JOINTS; + JointTracker* ljointTracker = editJointTracker(evalRealSenseJointIndex(rightSide, finger, finger_bone)); + if (jointData.confidence > 0) { + ljointTracker->editAbsFrame().setTranslation(vec3FromPXCPoint3DF32(jointData.positionWorld)); + ljointTracker->editAbsFrame().setRotation(quatFromPXCPoint4DF32(jointData.globalOrientation)); + ljointTracker->updateLocFromAbsTransform(parentJointTracker); + ljointTracker->activeFrame(); + } + if (finger_bone == (FINGER_NUM_JOINTS - 1)) { + parentJointTracker = palmJoint; + continue; + } + parentJointTracker = ljointTracker; + } + } + } + _manager->ReleaseFrame(); +#endif // HAVE_RSSDK +} + +void RealSense::loadRSSDKFile() { + QString fileNameString = QFileDialog::getOpenFileName(Application::getInstance()->getGLWidget(), tr("Open RSSDK clip"), + NULL, + tr("RSSDK Recordings (*.rssdk)")); + if (!fileNameString.isEmpty()) { + initSession(true, fileNameString); + } +} diff --git a/interface/src/devices/RealSense.h b/interface/src/devices/RealSense.h new file mode 100644 index 0000000000..015f1e7ce6 --- /dev/null +++ b/interface/src/devices/RealSense.h @@ -0,0 +1,63 @@ +// +// RealSense.h +// interface/src/devices +// +// Created by Thijs Wenker on 12/10/14 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_RealSense_h +#define hifi_RealSense_h + +#include + +#include "MotionTracker.h" + +#ifdef HAVE_RSSDK +#include +#include +#include +#include +#include +#endif + +/// Handles interaction with the RealSense skeleton tracking suit. +class RealSense : public QObject, public MotionTracker { + Q_OBJECT + +public: + static const Name NAME; + + static void init(); + + /// RealSense MotionTracker factory + static RealSense* getInstance(); + + bool isActive() const { return _active; } + + virtual void update(); + + void loadRSSDKFile(); + +protected: + RealSense(); + virtual ~RealSense(); + + void initSession(bool playback, QString filename); + +private: +#ifdef HAVE_RSSDK + PXCSession* _session; + PXCSenseManager* _manager; + PXCHandModule* _handController; + PXCHandData* _handData; + PXCHandConfiguration* _config; +#endif + bool _properlyInitialized; + bool _active; +}; + +#endif // hifi_RealSense_h From c4a9cae943c0efd659c6efb59d319e7c73f7ab16 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Mon, 15 Dec 2014 21:47:00 +0100 Subject: [PATCH 02/11] flipped some values to get a nicer hand-tracking --- interface/src/devices/RealSense.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/interface/src/devices/RealSense.cpp b/interface/src/devices/RealSense.cpp index 08492d8377..2c041a33c9 100644 --- a/interface/src/devices/RealSense.cpp +++ b/interface/src/devices/RealSense.cpp @@ -21,7 +21,7 @@ const DeviceTracker::Name RealSense::NAME = "RealSense"; // the side: true = right // the finger & the bone: // finger in [0..4] : bone in [0..3] a finger phalange -// [-1] up the hand branch : bone in [0..2] <=> [ hand, forearm, arm] +// [-1] up the hand branch : bone in [0..1] <=> [ hand, forearm] MotionTracker::Index evalRealSenseJointIndex(bool isRightSide, int finger, int bone) { MotionTracker::Index offset = 1 // start after root @@ -116,6 +116,7 @@ void RealSense::initSession(bool playback, QString filename) { _handData->Release(); _handController->Release(); _session->Release(); + _config->Release(); } _manager = _session->CreateSenseManager(); if (playback) { @@ -139,15 +140,20 @@ void RealSense::initSession(bool playback, QString filename) { } _properlyInitialized = true; } + + _config = _handController->CreateActiveConfiguration(); + _config->EnableStabilizer(true); + _config->SetTrackingMode(PXCHandData::TRACKING_MODE_FULL_HAND); + _config->ApplyChanges(); } #ifdef HAVE_RSSDK glm::quat quatFromPXCPoint4DF32(const PXCPoint4DF32& basis) { - return glm::quat(basis.w, basis.x, basis.y, basis.z); + return glm::normalize(glm::quat(basis.w, basis.x, basis.y, basis.z) * glm::quat(glm::vec3(0, M_PI, 0))); } glm::vec3 vec3FromPXCPoint3DF32(const PXCPoint3DF32& vec) { - return glm::vec3(vec.x, vec.y, vec.z); + return glm::vec3(-vec.x, vec.y, -vec.z); } #endif From 2633b0dee72d4e285bee3429c1b9eee6b0fec653 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Mon, 15 Dec 2014 22:26:47 +0100 Subject: [PATCH 03/11] reordered includes --- interface/src/Menu.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 9ef794db97..a9a039cd38 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -40,6 +40,7 @@ #include "AccountManager.h" #include "devices/Faceshift.h" #include "devices/OculusManager.h" +#include "devices/RealSense.h" #include "devices/Visage.h" #include "Menu.h" #include "scripting/LocationScriptingInterface.h" @@ -53,8 +54,6 @@ #include "ui/ModelsBrowser.h" #include "ui/LoginDialog.h" #include "ui/NodeBounds.h" -#include "devices/RealSense.h" -#include "devices/OculusManager.h" Menu* Menu::_instance = NULL; From 99b0cca8f32f4c93e8c5243dd8f855b598b6ff30 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Thu, 18 Dec 2014 00:08:20 +0100 Subject: [PATCH 04/11] realsenseHands.js example --- examples/realsenseHands.js | 525 +++++++++++++++++++++++++++++++++++++ 1 file changed, 525 insertions(+) create mode 100644 examples/realsenseHands.js diff --git a/examples/realsenseHands.js b/examples/realsenseHands.js new file mode 100644 index 0000000000..6b1a5c0a23 --- /dev/null +++ b/examples/realsenseHands.js @@ -0,0 +1,525 @@ +// +// realsenseHands.js +// examples +// +// Created by Thijs Wenker on 18 Dec 2014. +// Copyright 2014 High Fidelity, Inc. +// +// This is an example script that uses the Intel RealSense to make the avatar's hands replicate the user's hand actions. +// Most of this script is copied from leapHands.js +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +var leapHands = (function () { + + var isOnHMD, + MotionTracker = "RealSense", + LEAP_ON_HMD_MENU_ITEM = "Leap Motion on HMD", + LEAP_OFFSET = 0.019, // Thickness of Leap Motion plus HMD clip + HMD_OFFSET = 0.070, // Eyeballs to front surface of Oculus DK2 TODO: Confirm and make depend on device and eye relief + hasHandAndWristJoints, + handToWristOffset = [], // For avatars without a wrist joint we control an estimate of a proper hand joint position + HAND_OFFSET = 0.4, // Relative distance of wrist to hand versus wrist to index finger knuckle + hands, + wrists, + NUM_HANDS = 2, // 0 = left; 1 = right + fingers, + NUM_FINGERS = 5, // 0 = thumb; ...; 4 = pinky + THUMB = 0, + MIDDLE_FINGER = 2, + NUM_FINGER_JOINTS = 3, // 0 = metacarpal(hand)-proximal(finger) joint; ...; 2 = intermediate-distal joint + MAX_HAND_INACTIVE_COUNT = 20, + calibrationStatus, + UNCALIBRATED = 0, + CALIBRATING = 1, + CALIBRATED = 2, + CALIBRATION_TIME = 1000, // milliseconds + avatarScale, + avatarFaceModelURL, + avatarSkeletonModelURL, + settingsTimer; + + function printSkeletonJointNames() { + var jointNames, + i; + + print(MyAvatar.skeletonModelURL); + + print("Skeleton joint names ..."); + jointNames = MyAvatar.getJointNames(); + for (i = 0; i < jointNames.length; i += 1) { + print(i + ": " + jointNames[i]); + } + print("... skeleton joint names"); + + /* + http://public.highfidelity.io/models/skeletons/ron_standing.fst + Skeleton joint names ... + 0: Hips + 1: RightUpLeg + 2: RightLeg + 3: RightFoot + 4: RightToeBase + 5: RightToe_End + 6: LeftUpLeg + 7: LeftLeg + 8: LeftFoot + 9: LeftToeBase + 10: LeftToe_End + 11: Spine + 12: Spine1 + 13: Spine2 + 14: RightShoulder + 15: RightArm + 16: RightForeArm + 17: RightHand + 18: RightHandPinky1 + 19: RightHandPinky2 + 20: RightHandPinky3 + 21: RightHandPinky4 + 22: RightHandRing1 + 23: RightHandRing2 + 24: RightHandRing3 + 25: RightHandRing4 + 26: RightHandMiddle1 + 27: RightHandMiddle2 + 28: RightHandMiddle3 + 29: RightHandMiddle4 + 30: RightHandIndex1 + 31: RightHandIndex2 + 32: RightHandIndex3 + 33: RightHandIndex4 + 34: RightHandThumb1 + 35: RightHandThumb2 + 36: RightHandThumb3 + 37: RightHandThumb4 + 38: LeftShoulder + 39: LeftArm + 40: LeftForeArm + 41: LeftHand + 42: LeftHandPinky1 + 43: LeftHandPinky2 + 44: LeftHandPinky3 + 45: LeftHandPinky4 + 46: LeftHandRing1 + 47: LeftHandRing2 + 48: LeftHandRing3 + 49: LeftHandRing4 + 50: LeftHandMiddle1 + 51: LeftHandMiddle2 + 52: LeftHandMiddle3 + 53: LeftHandMiddle4 + 54: LeftHandIndex1 + 55: LeftHandIndex2 + 56: LeftHandIndex3 + 57: LeftHandIndex4 + 58: LeftHandThumb1 + 59: LeftHandThumb2 + 60: LeftHandThumb3 + 61: LeftHandThumb4 + 62: Neck + 63: Head + 64: HeadTop_End + 65: body + ... skeleton joint names + */ + } + + function finishCalibration() { + var avatarPosition, + handPosition, + middleFingerPosition, + leapHandHeight, + h; + + if (!isOnHMD) { + if (hands[0].controller.isActive() && hands[1].controller.isActive()) { + leapHandHeight = (hands[0].controller.getAbsTranslation().y + hands[1].controller.getAbsTranslation().y) / 2.0; + } else { + calibrationStatus = UNCALIBRATED; + return; + } + } + + avatarPosition = MyAvatar.position; + + for (h = 0; h < NUM_HANDS; h += 1) { + handPosition = MyAvatar.getJointPosition(hands[h].jointName); + if (!hasHandAndWristJoints) { + middleFingerPosition = MyAvatar.getJointPosition(fingers[h][MIDDLE_FINGER][0].jointName); + handToWristOffset[h] = Vec3.multiply(Vec3.subtract(handPosition, middleFingerPosition), 1.0 - HAND_OFFSET); + } + + if (isOnHMD) { + // Offset of Leap Motion origin from physical eye position + hands[h].zeroPosition = { x: 0.0, y: 0.0, z: HMD_OFFSET + LEAP_OFFSET }; + } else { + hands[h].zeroPosition = { + x: handPosition.x - avatarPosition.x, + y: handPosition.y - avatarPosition.y, + z: avatarPosition.z - handPosition.z + }; + hands[h].zeroPosition = Vec3.multiplyQbyV(MyAvatar.orientation, hands[h].zeroPosition); + hands[h].zeroPosition.y = hands[h].zeroPosition.y - leapHandHeight; + } + } + + MyAvatar.clearJointData("LeftHand"); + MyAvatar.clearJointData("LeftForeArm"); + MyAvatar.clearJointData("LeftArm"); + MyAvatar.clearJointData("RightHand"); + MyAvatar.clearJointData("RightForeArm"); + MyAvatar.clearJointData("RightArm"); + + calibrationStatus = CALIBRATED; + print("Leap Motion: Calibrated"); + } + + function calibrate() { + var jointNames, + i; + + calibrationStatus = CALIBRATING; + + avatarScale = MyAvatar.scale; + avatarFaceModelURL = MyAvatar.faceModelURL; + avatarSkeletonModelURL = MyAvatar.skeletonModelURL; + + // Does this skeleton have both wrist and hand joints? + hasHandAndWristJoints = false; + jointNames = MyAvatar.getJointNames(); + for (i = 0; i < jointNames.length; i += 1) { + hasHandAndWristJoints = hasHandAndWristJoints || jointNames[i].toLowerCase() === "leftwrist"; + } + + // Set avatar arms vertical, forearms horizontal, as "zero" position for calibration + MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0)); + MyAvatar.setJointData("LeftForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); + MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollRadians(0.0, 0.0, 0.0)); + MyAvatar.setJointData("RightArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 90.0)); + MyAvatar.setJointData("RightForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); + MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollRadians(0.0, 0.0, 0.0)); + + + // Wait for arms to assume their positions before calculating + Script.setTimeout(finishCalibration, CALIBRATION_TIME); + } + + function checkCalibration() { + + if (calibrationStatus === CALIBRATED) { + return true; + } + + if (calibrationStatus !== CALIBRATING) { + calibrate(); + } + + return false; + } + + function setIsOnHMD() { + isOnHMD = Menu.isOptionChecked(LEAP_ON_HMD_MENU_ITEM); + print("Leap Motion: " + (isOnHMD ? "Is on HMD" : "Is on desk")); + } + + function checkSettings() { + if (calibrationStatus > UNCALIBRATED && (MyAvatar.scale !== avatarScale + || MyAvatar.faceModelURL !== avatarFaceModelURL + || MyAvatar.skeletonModelURL !== avatarSkeletonModelURL + || Menu.isOptionChecked(LEAP_ON_HMD_MENU_ITEM) !== isOnHMD)) { + print("Leap Motion: Recalibrate..."); + calibrationStatus = UNCALIBRATED; + + setIsOnHMD(); + } + } + + function setUp() { + + wrists = [ + { + jointName: "LeftWrist", + controller: Controller.createInputController(MotionTracker, "joint_L_wrist") + }, + { + jointName: "RightWrist", + controller: Controller.createInputController(MotionTracker, "joint_R_wrist") + } + ]; + + hands = [ + { + jointName: "LeftHand", + controller: Controller.createInputController(MotionTracker, "joint_L_hand"), + inactiveCount: 0 + }, + { + jointName: "RightHand", + controller: Controller.createInputController(MotionTracker, "joint_R_hand"), + inactiveCount: 0 + } + ]; + + // The Leap controller's first joint is the hand-metacarpal joint but this joint's data is not used because it's too + // dependent on the model skeleton exactly matching the Leap skeleton; using just the second and subsequent joints + // seems to work better over all. + fingers = [{}, {}]; + fingers[0] = [ + [ + { jointName: "LeftHandThumb1", controller: Controller.createInputController(MotionTracker, "joint_L_thumb2") }, + { jointName: "LeftHandThumb2", controller: Controller.createInputController(MotionTracker, "joint_L_thumb3") }, + { jointName: "LeftHandThumb3", controller: Controller.createInputController(MotionTracker, "joint_L_thumb4") } + ], + [ + { jointName: "LeftHandIndex1", controller: Controller.createInputController(MotionTracker, "joint_L_index2") }, + { jointName: "LeftHandIndex2", controller: Controller.createInputController(MotionTracker, "joint_L_index3") }, + { jointName: "LeftHandIndex3", controller: Controller.createInputController(MotionTracker, "joint_L_index4") } + ], + [ + { jointName: "LeftHandMiddle1", controller: Controller.createInputController(MotionTracker, "joint_L_middle2") }, + { jointName: "LeftHandMiddle2", controller: Controller.createInputController(MotionTracker, "joint_L_middle3") }, + { jointName: "LeftHandMiddle3", controller: Controller.createInputController(MotionTracker, "joint_L_middle4") } + ], + [ + { jointName: "LeftHandRing1", controller: Controller.createInputController(MotionTracker, "joint_L_ring2") }, + { jointName: "LeftHandRing2", controller: Controller.createInputController(MotionTracker, "joint_L_ring3") }, + { jointName: "LeftHandRing3", controller: Controller.createInputController(MotionTracker, "joint_L_ring4") } + ], + [ + { jointName: "LeftHandPinky1", controller: Controller.createInputController(MotionTracker, "joint_L_pinky2") }, + { jointName: "LeftHandPinky2", controller: Controller.createInputController(MotionTracker, "joint_L_pinky3") }, + { jointName: "LeftHandPinky3", controller: Controller.createInputController(MotionTracker, "joint_L_pinky4") } + ] + ]; + fingers[1] = [ + [ + { jointName: "RightHandThumb1", controller: Controller.createInputController(MotionTracker, "joint_R_thumb2") }, + { jointName: "RightHandThumb2", controller: Controller.createInputController(MotionTracker, "joint_R_thumb3") }, + { jointName: "RightHandThumb3", controller: Controller.createInputController(MotionTracker, "joint_R_thumb4") } + ], + [ + { jointName: "RightHandIndex1", controller: Controller.createInputController(MotionTracker, "joint_R_index2") }, + { jointName: "RightHandIndex2", controller: Controller.createInputController(MotionTracker, "joint_R_index3") }, + { jointName: "RightHandIndex3", controller: Controller.createInputController(MotionTracker, "joint_R_index4") } + ], + [ + { jointName: "RightHandMiddle1", controller: Controller.createInputController(MotionTracker, "joint_R_middle2") }, + { jointName: "RightHandMiddle2", controller: Controller.createInputController(MotionTracker, "joint_R_middle3") }, + { jointName: "RightHandMiddle3", controller: Controller.createInputController(MotionTracker, "joint_R_middle4") } + ], + [ + { jointName: "RightHandRing1", controller: Controller.createInputController(MotionTracker, "joint_R_ring2") }, + { jointName: "RightHandRing2", controller: Controller.createInputController(MotionTracker, "joint_R_ring3") }, + { jointName: "RightHandRing3", controller: Controller.createInputController(MotionTracker, "joint_R_ring4") } + ], + [ + { jointName: "RightHandPinky1", controller: Controller.createInputController(MotionTracker, "joint_R_pinky2") }, + { jointName: "RightHandPinky2", controller: Controller.createInputController(MotionTracker, "joint_R_pinky3") }, + { jointName: "RightHandPinky3", controller: Controller.createInputController(MotionTracker, "joint_R_pinky4") } + ] + ]; + + setIsOnHMD(); + + settingsTimer = Script.setInterval(checkSettings, 2000); + + calibrationStatus = UNCALIBRATED; + } + + function moveHands() { + var h, + i, + j, + side, + handOffset, + wristOffset, + handRotation, + locRotation, + cameraOrientation, + inverseAvatarOrientation; + + for (h = 0; h < NUM_HANDS; h += 1) { + side = h === 0 ? -1.0 : 1.0; + + if (hands[h].controller.isActive()) { + + // Calibrate if necessary. + if (!checkCalibration()) { + return; + } + + // Hand position ... + handOffset = hands[h].controller.getAbsTranslation(); + handRotation = hands[h].controller.getAbsRotation(); + + if (isOnHMD) { + + // Adjust to control wrist position if "hand" joint is at wrist ... + if (!hasHandAndWristJoints) { + wristOffset = Vec3.multiplyQbyV(handRotation, handToWristOffset[h]); + handOffset = Vec3.sum(handOffset, wristOffset); + } + + // Hand offset in camera coordinates ... + handOffset = { + x: hands[h].zeroPosition.x - handOffset.x, + y: hands[h].zeroPosition.y - handOffset.z, + z: hands[h].zeroPosition.z + handOffset.y + }; + handOffset.z = -handOffset.z; + + // Hand offset in world coordinates ... + cameraOrientation = Camera.getOrientation(); + handOffset = Vec3.sum(Camera.getPosition(), Vec3.multiplyQbyV(cameraOrientation, handOffset)); + + // Hand offset in avatar coordinates ... + inverseAvatarOrientation = Quat.inverse(MyAvatar.orientation); + handOffset = Vec3.subtract(handOffset, MyAvatar.position); + handOffset = Vec3.multiplyQbyV(inverseAvatarOrientation, handOffset); + handOffset.z = -handOffset.z; + handOffset.x = -handOffset.x; + + // Hand rotation in camera coordinates ... + handRotation = { + x: handRotation.z, + y: handRotation.y, + z: handRotation.x, + w: handRotation.w + }; + + // Hand rotation in avatar coordinates ... + if (h === 0) { + handRotation.x = -handRotation.x; + handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 1, y: 0, z: 0 }), handRotation); + handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 0, y: 0, z: 1 }), handRotation); + } else { + handRotation.z = -handRotation.z; + handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 1, y: 0, z: 0 }), handRotation); + handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 0, y: 0, z: 1 }), handRotation); + } + + cameraOrientation.x = -cameraOrientation.x; + cameraOrientation.z = -cameraOrientation.z; + handRotation = Quat.multiply(cameraOrientation, handRotation); + handRotation = Quat.multiply(inverseAvatarOrientation, handRotation); + + } else { + + // Hand offset in camera coordinates ... + handOffset = { + x: -handOffset.x, + y: hands[h].zeroPosition.y + handOffset.y, + z: hands[h].zeroPosition.z - handOffset.z + }; + + // Hand rotation in camera coordinates ... + handRotation = { + x: handRotation.y, + y: handRotation.z, + z: -handRotation.x, + w: handRotation.w + }; + + // Hand rotation in avatar coordinates ... + if (h === 0) { + handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 0, y: 1, z: 0 }), + handRotation); + handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 1, y: 0, z: 0 }), + handRotation); + } else { + handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 0, y: 1, z: 0 }), + handRotation); + handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 1, y: 0, z: 0 }), + handRotation); + } + } + + // Set hand position and orientation ... + MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset, handRotation, true); + + // Set finger joints ... + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (fingers[h][i][j].controller !== null) { + locRotation = fingers[h][i][j].controller.getLocRotation(); + if (i === THUMB) { + locRotation = { + x: side * locRotation.y, + y: side * -locRotation.z, + z: side * -locRotation.x, + w: locRotation.w + }; + } else { + locRotation = { + x: -locRotation.x, + y: -locRotation.z, + z: -locRotation.y, + w: locRotation.w + }; + } + MyAvatar.setJointData(fingers[h][i][j].jointName, locRotation); + } + } + } + + hands[h].inactiveCount = 0; + + } else { + + if (hands[h].inactiveCount < MAX_HAND_INACTIVE_COUNT) { + + hands[h].inactiveCount += 1; + + if (hands[h].inactiveCount === MAX_HAND_INACTIVE_COUNT) { + if (h === 0) { + MyAvatar.clearJointData("LeftHand"); + MyAvatar.clearJointData("LeftForeArm"); + MyAvatar.clearJointData("LeftArm"); + } else { + MyAvatar.clearJointData("RightHand"); + MyAvatar.clearJointData("RightForeArm"); + MyAvatar.clearJointData("RightArm"); + } + } + } + } + } + } + + function tearDown() { + var h, + i, + j; + + Script.clearInterval(settingsTimer); + + for (h = 0; h < NUM_HANDS; h += 1) { + Controller.releaseInputController(hands[h].controller); + Controller.releaseInputController(wrists[h].controller); + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (fingers[h][i][j].controller !== null) { + Controller.releaseInputController(fingers[h][i][j].controller); + } + } + } + } + } + + return { + printSkeletonJointNames: printSkeletonJointNames, + setUp : setUp, + moveHands : moveHands, + tearDown : tearDown + }; +}()); + + +//leapHands.printSkeletonJointNames(); + +leapHands.setUp(); +Script.update.connect(leapHands.moveHands); +Script.scriptEnding.connect(leapHands.tearDown); From 151ddab118c4a95b437711f78a83dbe862cdf27c Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Thu, 18 Dec 2014 00:16:51 +0100 Subject: [PATCH 05/11] make stuff build without RealSense --- interface/src/devices/RealSense.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/interface/src/devices/RealSense.cpp b/interface/src/devices/RealSense.cpp index 2c041a33c9..b09c165e44 100644 --- a/interface/src/devices/RealSense.cpp +++ b/interface/src/devices/RealSense.cpp @@ -23,7 +23,7 @@ const DeviceTracker::Name RealSense::NAME = "RealSense"; // finger in [0..4] : bone in [0..3] a finger phalange // [-1] up the hand branch : bone in [0..1] <=> [ hand, forearm] MotionTracker::Index evalRealSenseJointIndex(bool isRightSide, int finger, int bone) { - +#ifdef HAVE_RSSDK MotionTracker::Index offset = 1 // start after root + (int(isRightSide) * PXCHandData::NUMBER_OF_JOINTS) // then offset for side + PALMROOT_NUM_JOINTS; // then add the arm/forearm/hand chain @@ -34,6 +34,9 @@ MotionTracker::Index evalRealSenseJointIndex(bool isRightSide, int finger, int b // or go back up for the correct root bone return offset - 1 - bone; } +#else + return -1; +#endif // HAVE_RSSDK } // static @@ -62,6 +65,7 @@ RealSense::RealSense() : _active(false), _handData(NULL) { +#ifdef HAVE_RSSDK _session = PXCSession_Create(); initSession(false, NULL); @@ -103,13 +107,17 @@ RealSense::RealSense() : } } } +#endif // HAVE_RSSDK } RealSense::~RealSense() { +#ifdef HAVE_RSSDK _manager->Release(); +#endif // HAVE_RSSDK } void RealSense::initSession(bool playback, QString filename) { +#ifdef HAVE_RSSDK _active = false; _properlyInitialized = false; if (_handData != NULL) { @@ -145,6 +153,7 @@ void RealSense::initSession(bool playback, QString filename) { _config->EnableStabilizer(true); _config->SetTrackingMode(PXCHandData::TRACKING_MODE_FULL_HAND); _config->ApplyChanges(); +#endif // HAVE_RSSDK } #ifdef HAVE_RSSDK @@ -155,7 +164,7 @@ glm::quat quatFromPXCPoint4DF32(const PXCPoint4DF32& basis) { glm::vec3 vec3FromPXCPoint3DF32(const PXCPoint3DF32& vec) { return glm::vec3(-vec.x, vec.y, -vec.z); } -#endif +#endif // HAVE_RSSDK void RealSense::update() { #ifdef HAVE_RSSDK From 8a1618c26934af2645fb3d178e2e42d09d769426 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Thu, 18 Dec 2014 00:22:59 +0100 Subject: [PATCH 06/11] missed one. --- interface/src/devices/RealSense.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/devices/RealSense.cpp b/interface/src/devices/RealSense.cpp index b09c165e44..5edb3aacee 100644 --- a/interface/src/devices/RealSense.cpp +++ b/interface/src/devices/RealSense.cpp @@ -62,10 +62,10 @@ RealSense* RealSense::getInstance() { RealSense::RealSense() : MotionTracker(), - _active(false), - _handData(NULL) + _active(false) { #ifdef HAVE_RSSDK + _handData = NULL; _session = PXCSession_Create(); initSession(false, NULL); From 95b0f2515b327e38cbd6d0dcb18b0b8a77d8b635 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Thu, 18 Dec 2014 00:32:11 +0100 Subject: [PATCH 07/11] this doesn't make `real sense` to me, perhaps this helps --- interface/src/devices/RealSense.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/devices/RealSense.cpp b/interface/src/devices/RealSense.cpp index 5edb3aacee..c2591772d0 100644 --- a/interface/src/devices/RealSense.cpp +++ b/interface/src/devices/RealSense.cpp @@ -239,7 +239,7 @@ void RealSense::update() { } void RealSense::loadRSSDKFile() { - QString fileNameString = QFileDialog::getOpenFileName(Application::getInstance()->getGLWidget(), tr("Open RSSDK clip"), + QString fileNameString = QFileDialog::getOpenFileName(Application::getInstance()->getWindow(), tr("Open RSSDK clip"), NULL, tr("RSSDK Recordings (*.rssdk)")); if (!fileNameString.isEmpty()) { From 835e14a7b2190c807b95782939cf642378e91434 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Thu, 18 Dec 2014 00:52:31 +0100 Subject: [PATCH 08/11] This might solve it? --- interface/src/devices/RealSense.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/devices/RealSense.cpp b/interface/src/devices/RealSense.cpp index c2591772d0..aea7a50779 100644 --- a/interface/src/devices/RealSense.cpp +++ b/interface/src/devices/RealSense.cpp @@ -239,8 +239,9 @@ void RealSense::update() { } void RealSense::loadRSSDKFile() { + QString locationDir(QStandardPaths::displayName(QStandardPaths::DesktopLocation)); QString fileNameString = QFileDialog::getOpenFileName(Application::getInstance()->getWindow(), tr("Open RSSDK clip"), - NULL, + locationDir, tr("RSSDK Recordings (*.rssdk)")); if (!fileNameString.isEmpty()) { initSession(true, fileNameString); From 2884de1fe4f6493a888ce7601e7f97994fbe8b0d Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Thu, 18 Dec 2014 01:18:05 +0100 Subject: [PATCH 09/11] include fix --- interface/src/devices/RealSense.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/interface/src/devices/RealSense.cpp b/interface/src/devices/RealSense.cpp index aea7a50779..f082f96949 100644 --- a/interface/src/devices/RealSense.cpp +++ b/interface/src/devices/RealSense.cpp @@ -9,6 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // #include "RealSense.h" +#include "MainWindow.h" #include "Menu.h" #include "SharedUtil.h" From f67df35f03f5597146da55d06ae917c7bdc9216c Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Fri, 19 Dec 2014 16:36:06 +0100 Subject: [PATCH 10/11] rssdk instructions --- interface/external/rssdk/readme.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 interface/external/rssdk/readme.txt diff --git a/interface/external/rssdk/readme.txt b/interface/external/rssdk/readme.txt new file mode 100644 index 0000000000..fe2246e32a --- /dev/null +++ b/interface/external/rssdk/readme.txt @@ -0,0 +1,9 @@ + +Instructions for adding the Intel Realsense (RSSDK) to Interface +Thijs Wenker, December 19, 2014 + +This is Windows only for now. + +1. Download the SDK at https://software.intel.com/en-us/intel-realsense-sdk/download + +2. Copy the `lib` and `include` folder inside this directory \ No newline at end of file From 8ffea45736c3b7cbc2abed12e4473f4ab5a35ae4 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Thu, 15 Jan 2015 02:59:37 +0100 Subject: [PATCH 11/11] QFileDialog include --- interface/src/devices/RealSense.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/devices/RealSense.h b/interface/src/devices/RealSense.h index 015f1e7ce6..4a43a1e117 100644 --- a/interface/src/devices/RealSense.h +++ b/interface/src/devices/RealSense.h @@ -12,7 +12,7 @@ #ifndef hifi_RealSense_h #define hifi_RealSense_h -#include +#include #include "MotionTracker.h"