From b4f5df17ab9d22a9985ff0d5972d9f80b17ce782 Mon Sep 17 00:00:00 2001 From: Eric Johnston Date: Thu, 27 Jun 2013 10:54:36 -0700 Subject: [PATCH] (re-commit) Added support for Leap finger-sensing device. Note that the actual Leap SDK can't be in the repo for IP reasons, so there's a stub-version of the header. When the actual SDK is put into the Leap folder, cake will find it automatically and switch over. --- cmake/modules/FindLeap.cmake | 52 ++++++++++++ interface/CMakeLists.txt | 4 + interface/external/Leap/stubs/include/Leap.h | 13 +++ interface/src/Application.cpp | 8 +- interface/src/Avatar.cpp | 60 +++++++++++++ interface/src/Avatar.h | 11 ++- interface/src/LeapManager.cpp | 88 ++++++++++++++++++++ interface/src/LeapManager.h | 39 +++++++++ 8 files changed, 273 insertions(+), 2 deletions(-) create mode 100755 cmake/modules/FindLeap.cmake mode change 100644 => 100755 interface/CMakeLists.txt create mode 100755 interface/external/Leap/stubs/include/Leap.h mode change 100644 => 100755 interface/src/Application.cpp mode change 100644 => 100755 interface/src/Avatar.cpp mode change 100644 => 100755 interface/src/Avatar.h create mode 100755 interface/src/LeapManager.cpp create mode 100755 interface/src/LeapManager.h diff --git a/cmake/modules/FindLeap.cmake b/cmake/modules/FindLeap.cmake new file mode 100755 index 0000000000..121d422131 --- /dev/null +++ b/cmake/modules/FindLeap.cmake @@ -0,0 +1,52 @@ +# - Try to find the Leap library +# +# You must provide a LEAP_ROOT_DIR which contains Lib and Include directories +# +# Once done this will define +# +# LEAP_FOUND - system found Leap +# LEAP_INCLUDE_DIRS - the Leap include directory +# LEAP_LIBRARIES - Link this to use Leap +# +# Created on 6/21/2013 by Eric Johnston, +# adapted from FindLibOVR.cmake by Stephen Birarda +# Copyright (c) 2013 High Fidelity +# + +if (LEAP_LIBRARIES AND LEAP_INCLUDE_DIRS) + # in cache already + set(LEAP_FOUND TRUE) +else (LEAP_LIBRARIES AND LEAP_INCLUDE_DIRS) + find_path(LEAP_INCLUDE_DIRS Leap.h ${LEAP_ROOT_DIR}/include) + + if (LEAP_INCLUDE_DIRS) + ### If we found the real header, get the library as well. + if (APPLE) + find_library(LEAP_LIBRARIES libLeap.dylib ${LEAP_ROOT_DIR}/lib/libc++/) + elseif (WIN32) + find_library(LEAP_LIBRARIES libLeap.dylib ${LEAP_ROOT_DIR}/lib/libc++/) + endif () + else () + ### If we didn't find the real header, just use the stub header, and no library. + find_path(LEAP_INCLUDE_DIRS Leap.h ${LEAP_ROOT_DIR}/stubs/include) + endif () + +# If we're using the Leap stubs, there's only a header, no lib. + if (LEAP_INCLUDE_DIRS) + set(LEAP_FOUND TRUE) + endif (LEAP_INCLUDE_DIRS) + + if (LEAP_FOUND) + if (NOT Leap_FIND_QUIETLY) + message(STATUS "Found Leap: ${LEAP_LIBRARIES}") + endif (NOT Leap_FIND_QUIETLY) + else (LEAP_FOUND) + if (Leap_FIND_REQUIRED) + message(FATAL_ERROR "Could not find Leap") + endif (Leap_FIND_REQUIRED) + endif (LEAP_FOUND) + + # show the LEAP_INCLUDE_DIRS and LEAP_LIBRARIES variables only in the advanced view + mark_as_advanced(LEAP_INCLUDE_DIRS LEAP_LIBRARIES) + +endif (LEAP_LIBRARIES AND LEAP_INCLUDE_DIRS) \ No newline at end of file diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt old mode 100644 new mode 100755 index 9a57a546f1..c840bd980d --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -9,6 +9,7 @@ project(${TARGET_NAME}) # setup for find modules set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../cmake/modules/") set(LIBOVR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/LibOVR) +set(LEAP_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/Leap) set(PORTAUDIO_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/PortAudio) set(OPENCV_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/OpenCV) set(UVCCAMERACONTROL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/UVCCameraControl) @@ -86,6 +87,7 @@ link_hifi_library(audio ${TARGET_NAME} ${ROOT_DIR}) # find required libraries find_package(GLM REQUIRED) find_package(LibOVR) +find_package(Leap) find_package(OpenCV) find_package(ZLIB) find_package(UVCCameraControl) @@ -102,6 +104,7 @@ include_directories( SYSTEM ${GLM_INCLUDE_DIRS} ${LIBOVR_INCLUDE_DIRS} + ${LEAP_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ) @@ -140,6 +143,7 @@ if (APPLE) ${QuartzCore} ${UVCCAMERACONTROL_LIBRARIES} ${LIBOVR_LIBRARIES} + ${LEAP_LIBRARIES} ) else (APPLE) find_package(OpenGL REQUIRED) diff --git a/interface/external/Leap/stubs/include/Leap.h b/interface/external/Leap/stubs/include/Leap.h new file mode 100755 index 0000000000..6d012b9e98 --- /dev/null +++ b/interface/external/Leap/stubs/include/Leap.h @@ -0,0 +1,13 @@ + +// This is an empty stub, used as a placeholder for the real Leap.h +// The entire containing Leap folder should be replaced by the one +// from the Leap SDK. + +#define LEAP_STUBS // We're using the stubbed-out Leap header + +namespace Leap { + class Frame {}; + class Controller {}; + class Listener {}; +} + diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp old mode 100644 new mode 100755 index 59c4215cfe..1d2fb990e5 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -57,6 +57,7 @@ #include "Application.h" #include "InterfaceConfig.h" #include "LogDisplay.h" +#include "LeapManager.h" #include "OculusManager.h" #include "Util.h" #include "renderer/ProgramObject.h" @@ -1671,7 +1672,11 @@ void Application::update(float deltaTime) { _touchAvgY - _touchDragStartedAvgY); } - // Read serial port interface devices + // Leap finger-sensing device + LeapManager::nextFrame(); + _myAvatar.setLeapFingers(LeapManager::getFingerPositions()); + + // Read serial port interface devices if (_serialHeadSensor.isActive()) { _serialHeadSensor.readData(deltaTime); } @@ -2315,6 +2320,7 @@ void Application::displayStats() { } drawtext(10, statsVerticalOffset + 330, 0.10f, 0, 1.0, 0, avatarMixerStats); + drawtext(10, statsVerticalOffset + 450, 0.10f, 0, 1.0, 0, (char *)LeapManager::statusString().c_str()); if (_perfStatsOn) { // Get the PerfStats group details. We need to allocate and array of char* long enough to hold 1+groups diff --git a/interface/src/Avatar.cpp b/interface/src/Avatar.cpp old mode 100644 new mode 100755 index a603dbd472..98b3ad06a1 --- a/interface/src/Avatar.cpp +++ b/interface/src/Avatar.cpp @@ -104,6 +104,7 @@ Avatar::Avatar(Agent* owningAgent) : _skeleton.initialize(); initializeBodyBalls(); + initializeLeapBalls(); _height = _skeleton.getHeight() + _bodyBall[ BODY_BALL_LEFT_HEEL ].radius + _bodyBall[ BODY_BALL_HEAD_BASE ].radius; @@ -263,6 +264,21 @@ void Avatar::initializeBodyBalls() { */ } +void Avatar::initializeLeapBalls() { + + _numLeapBalls = 0; + + for (int b = 0; b < MAX_AVATAR_LEAP_BALLS; b++) { + _leapBall[b].parentJoint = AVATAR_JOINT_NULL; + _leapBall[b].parentOffset = glm::vec3(0.0, 0.0, 0.0); + _leapBall[b].position = glm::vec3(0.0, 0.0, 0.0); + _leapBall[b].velocity = glm::vec3(0.0, 0.0, 0.0); + _leapBall[b].radius = 0.01; + _leapBall[b].touchForce = 0.0; + _leapBall[b].isCollidable = true; + _leapBall[b].jointTightness = BODY_SPRING_DEFAULT_TIGHTNESS; + } +} Avatar::~Avatar() { _headData = NULL; @@ -527,6 +543,9 @@ void Avatar::simulate(float deltaTime, Transmitter* transmitter) { // update body balls updateBodyBalls(deltaTime); + // update leap balls + updateLeapBalls(deltaTime); + // test for avatar collision response with the big sphere if (usingBigSphereCollisionTest) { updateCollisionWithSphere(_TEST_bigSpherePosition, _TEST_bigSphereRadius, deltaTime); @@ -1148,6 +1167,23 @@ void Avatar::updateBodyBalls(float deltaTime) { _bodyBall[BODY_BALL_HEAD_TOP].rotation * _skeleton.joint[BODY_BALL_HEAD_TOP].bindPosePosition; } +void Avatar::setLeapFingers(const std::vector& fingerPositions) { + _numLeapBalls = fingerPositions.size(); // just to test + + float unitScale = 0.001; // convert mm to meters + glm::vec3 offset(0.2, -0.2, -0.3); // place the hand in front of the face where we can see it + + for (int b = 0; b < _numLeapBalls; b++) { + glm::vec3 pos = unitScale * fingerPositions[b] + offset; + _leapBall[b].rotation = _head.getOrientation(); + _leapBall[b].position = _bodyBall[BODY_BALL_HEAD_BASE].position + + _head.getOrientation() * pos; + } +} + +void Avatar::updateLeapBalls(float deltaTime) { +} + void Avatar::updateArmIKAndConstraints(float deltaTime) { // determine the arm vector @@ -1270,6 +1306,30 @@ void Avatar::renderBody(bool lookingInMirror, bool renderAvatarBalls) { } } } + // Draw the leap balls + for (int b = 0; b < _numLeapBalls; b++) { + float alpha = 1.0f; + + if (alpha > 0.0f) { + // Render the body ball sphere + if (_owningAgent || true) { + glColor3f(SKIN_COLOR[0] + _leapBall[b].touchForce * 0.3f, + SKIN_COLOR[1] - _leapBall[b].touchForce * 0.2f, + SKIN_COLOR[2] - _leapBall[b].touchForce * 0.1f); + } else { + glColor4f(SKIN_COLOR[0] + _leapBall[b].touchForce * 0.3f, + SKIN_COLOR[1] - _leapBall[b].touchForce * 0.2f, + SKIN_COLOR[2] - _leapBall[b].touchForce * 0.1f, + alpha); + } + glColor4f(0.0, 0.4, 0.0, 1.0); // Just to test + + glPushMatrix(); + glTranslatef(_leapBall[b].position.x, _leapBall[b].position.y, _leapBall[b].position.z); + glutSolidSphere(_leapBall[b].radius, 20.0f, 20.0f); + glPopMatrix(); + } + } } else { // Render the body's voxels float alpha = getBallRenderAlpha(BODY_BALL_HEAD_BASE, lookingInMirror); diff --git a/interface/src/Avatar.h b/interface/src/Avatar.h old mode 100644 new mode 100755 index b7d424de29..e8cc2f5029 --- a/interface/src/Avatar.h +++ b/interface/src/Avatar.h @@ -56,6 +56,8 @@ enum AvatarBodyBallID NUM_AVATAR_BODY_BALLS }; +#define MAX_AVATAR_LEAP_BALLS 10 + enum DriveKeys { FWD = 0, @@ -103,6 +105,7 @@ public: void setGravity (glm::vec3 gravity); void setMouseRay (const glm::vec3 &origin, const glm::vec3 &direction); void setOrientation (const glm::quat& orientation); + void setLeapFingers (const std::vector& fingerPositions); //getters bool isInitialized () const { return _initialized;} @@ -111,7 +114,9 @@ public: float getBodyYaw () const { return _bodyYaw;} bool getIsNearInteractingOther () const { return _avatarTouch.getAbleToReachOtherAvatar();} const glm::vec3& getHeadJointPosition () const { return _skeleton.joint[ AVATAR_JOINT_HEAD_BASE ].position;} - const glm::vec3& getBallPosition (AvatarJointID j) const { return _bodyBall[j].position;} + const glm::vec3& getBallPosition (AvatarJointID j) const { return _bodyBall[j].position;} + int getNumLeapBalls () const { return _numLeapBalls;} + const glm::vec3& getLeapBallPosition (int which) const { return _leapBall[which].position;} glm::vec3 getBodyRightDirection () const { return getOrientation() * IDENTITY_RIGHT; } glm::vec3 getBodyUpDirection () const { return getOrientation() * IDENTITY_UP; } glm::vec3 getBodyFrontDirection () const { return getOrientation() * IDENTITY_FRONT; } @@ -185,6 +190,8 @@ private: float _bodyRollDelta; glm::vec3 _movedHandOffset; AvatarBall _bodyBall[ NUM_AVATAR_BODY_BALLS ]; + AvatarBall _leapBall[ MAX_AVATAR_LEAP_BALLS ]; + int _numLeapBalls; AvatarMode _mode; glm::vec3 _handHoldingPosition; glm::vec3 _velocity; @@ -219,8 +226,10 @@ private: float getBallRenderAlpha(int ball, bool lookingInMirror) const; void renderBody(bool lookingInMirror, bool renderAvatarBalls); void initializeBodyBalls(); + void initializeLeapBalls(); void resetBodyBalls(); void updateBodyBalls( float deltaTime ); + void updateLeapBalls( float deltaTime ); void calculateBoneLengths(); void readSensors(); void updateHandMovementAndTouching(float deltaTime); diff --git a/interface/src/LeapManager.cpp b/interface/src/LeapManager.cpp new file mode 100755 index 0000000000..2c2ce9b8d1 --- /dev/null +++ b/interface/src/LeapManager.cpp @@ -0,0 +1,88 @@ +// +// LeapManager.cpp +// hifi +// +// Created by Eric Johnston on 6/26/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include "LeapManager.h" +#include +#include // needed for RTLD_LAZY +#include + +bool LeapManager::_isInitialized = false; +bool LeapManager::_libraryExists = false; +Leap::Controller* LeapManager::_controller = 0; +HifiLeapListener* LeapManager::_listener = 0; + +class HifiLeapListener : public Leap::Listener { +public: + Leap::Frame lastFrame; + std::vector fingerPositions; + + virtual void onFrame(const Leap::Controller& controller) { +#ifndef LEAP_STUBS + Leap::Frame frame = controller.frame(); + int numFingers = frame.fingers().count(); + fingerPositions.resize(numFingers); + for (int i = 0; i < numFingers; ++i) { + const Leap::Finger& thisFinger = frame.fingers()[i]; + const Leap::Vector pos = thisFinger.tipPosition(); + fingerPositions[i] = glm::vec3(pos.x, pos.y, pos.z); + } + lastFrame = frame; +#endif + } + +}; + +void LeapManager::initialize() { + if (!_isInitialized) { +#ifndef LEAP_STUBS + if (dlopen("/usr/lib/libLeap.dylib", RTLD_LAZY)) { + _libraryExists = true; + _controller = new Leap::Controller(); + _listener = new HifiLeapListener(); + _controller->addListener(*_listener); + } +#endif + _isInitialized = true; + } +} + +void LeapManager::nextFrame() { + initialize(); + if (_listener && _controller) + _listener->onFrame(*_controller); +} + +const std::vector& LeapManager::getFingerPositions() { + if (_listener) + return _listener->fingerPositions; + else { + static std::vector empty; + return empty; + } +} + +std::string LeapManager::statusString() { + std::stringstream leapString; +#ifndef LEAP_STUBS + if (_isInitialized) { + if (!_libraryExists) + leapString << "Leap library at /usr/lib/libLeap.dylib does not exist."; + else if (!_controller || !_listener || !_controller->devices().count()) + leapString << "Leap controller is not attached."; + else { + leapString << "Leap pointables: " << _listener->lastFrame.pointables().count(); + if (_listener->lastFrame.pointables().count() > 0) { + Leap::Vector pos = _listener->lastFrame.pointables()[0].tipPosition(); + leapString << " pos: " << pos.x << " " << pos.y << " " << pos.z; + } + } + } +#endif + return leapString.str(); +} + diff --git a/interface/src/LeapManager.h b/interface/src/LeapManager.h new file mode 100755 index 0000000000..f09bbb8b88 --- /dev/null +++ b/interface/src/LeapManager.h @@ -0,0 +1,39 @@ +// +// LeapManager.h +// hifi +// +// Created by Eric Johnston on 6/26/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __hifi__LeapManager__ +#define __hifi__LeapManager__ + +#include +#include +#include + +class HifiLeapListener; +namespace Leap { + class Controller; +} + +class LeapManager { +public: + static void nextFrame(); // called once per frame to get new Leap data + static const std::vector& getFingerPositions(); + static std::string statusString(); + +private: + static void initialize(); + static bool _isInitialized; // We've looked for the library and hooked it up if it's there. + static bool _libraryExists; // The library is present, so we won't crash if we call it. + static Leap::Controller* _controller; + static HifiLeapListener* _listener; + + + + +}; + +#endif /* defined(__hifi__LeapManager__) */