From 5d59bcbec7e8ddceab4700f09d6416cd93518d07 Mon Sep 17 00:00:00 2001 From: Eric Johnston Date: Tue, 2 Jul 2013 17:03:32 -0700 Subject: [PATCH 1/2] (changed per requests and re-merged) Two Leap-related crashes fixed, plus Leap improvements 1. Fixed thread-crash at terminate() time by terminating the LeapManager properly 2. Fixed an intermittent thread-crash when Leap active by removing auto-connection between controller and listener 3. Added fingerRoot positions to display and data packet 4. Introduced a vec3-based convention so that more finger data may be added without causing trouble for old clients and servers 5. My avatar's fingers are different color from others --- interface/src/Application.cpp | 7 +- interface/src/Avatar.h | 6 +- interface/src/Hand.cpp | 77 +++++++++++++------ interface/src/Hand.h | 13 ++-- interface/src/LeapManager.cpp | 111 +++++++++++++++++++-------- interface/src/LeapManager.h | 13 ++-- libraries/avatars/src/AvatarData.cpp | 56 +++++++++----- libraries/avatars/src/HandData.h | 17 ++-- 8 files changed, 208 insertions(+), 92 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 70f7287aab..0e2ec13099 100755 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -953,6 +953,8 @@ void Application::terminate() { // Close serial port // close(serial_fd); + LeapManager::terminate(); + if (_settingsAutosave->isChecked()) { saveSettings(); _settings->sync(); @@ -1645,6 +1647,8 @@ void Application::init() { QMetaObject::invokeMethod(_fullScreenMode, "trigger", Qt::QueuedConnection); } + LeapManager::initialize(); + gettimeofday(&_timerStart, NULL); gettimeofday(&_lastTimeIdle, NULL); @@ -1803,7 +1807,8 @@ void Application::update(float deltaTime) { // Leap finger-sensing device LeapManager::nextFrame(); - _myAvatar.getHand().setLeapFingers(LeapManager::getFingerPositions()); + _myAvatar.getHand().setLeapFingers(LeapManager::getFingerTips(), LeapManager::getFingerRoots()); + _myAvatar.getHand().setLeapHands(LeapManager::getHandPositions(), LeapManager::getHandNormals()); // Read serial port interface devices if (_serialHeadSensor.isActive()) { diff --git a/interface/src/Avatar.h b/interface/src/Avatar.h index 86c333dad4..ec41d6d0eb 100755 --- a/interface/src/Avatar.h +++ b/interface/src/Avatar.h @@ -107,6 +107,7 @@ public: //getters bool isInitialized () const { return _initialized;} + bool isMyAvatar () const { return _owningAgent == NULL; } const Skeleton& getSkeleton () const { return _skeleton;} float getHeadYawRate () const { return _head.yawRate;} float getBodyYaw () const { return _bodyYaw;} @@ -155,13 +156,13 @@ public: void writeAvatarDataToFile(); void readAvatarDataFromFile(); + static void renderJointConnectingCone(glm::vec3 position1, glm::vec3 position2, float radius1, float radius2); + private: // privatize copy constructor and assignment operator to avoid copying Avatar(const Avatar&); Avatar& operator= (const Avatar&); - bool isMyAvatar() const { return _owningAgent == NULL; } - struct AvatarBall { AvatarJointID parentJoint; // the skeletal joint that serves as a reference for determining the position @@ -237,7 +238,6 @@ private: void applyHardCollision(const glm::vec3& penetration, float elasticity, float damping); void applyCollisionWithOtherAvatar( Avatar * other, float deltaTime ); void checkForMouseRayTouching(); - void renderJointConnectingCone(glm::vec3 position1, glm::vec3 position2, float radius1, float radius2); }; #endif diff --git a/interface/src/Hand.cpp b/interface/src/Hand.cpp index 5a44546d7b..ab44464a55 100755 --- a/interface/src/Hand.cpp +++ b/interface/src/Hand.cpp @@ -21,21 +21,18 @@ Hand::Hand(Avatar* owningAvatar) : _owningAvatar(owningAvatar), _renderAlpha(1.0), _lookingInMirror(false), - _ballColor(0.0, 0.4, 0.0), + _ballColor(0.0, 0.0, 0.4), _position(0.0, 0.4, 0.0), _orientation(0.0, 0.0, 0.0, 1.0) { } void Hand::init() { - _numLeapBalls = 0; - for (int b = 0; b < MAX_AVATAR_LEAP_BALLS; b++) { - _leapBall[b].position = glm::vec3(0.0, 0.0, 0.0); - _leapBall[b].velocity = glm::vec3(0.0, 0.0, 0.0); - _leapBall[b].radius = 0.01; - _leapBall[b].touchForce = 0.0; - _leapBall[b].isCollidable = true; - } + // Different colors for my hand and others' hands + if (_owningAvatar && _owningAvatar->isMyAvatar()) + _ballColor = glm::vec3(0.0, 0.4, 0.0); + else + _ballColor = glm::vec3(0.0, 0.0, 0.4); } void Hand::reset() { @@ -44,20 +41,27 @@ void Hand::reset() { void Hand::simulate(float deltaTime, bool isMine) { } +glm::vec3 Hand::leapPositionToWorldPosition(const glm::vec3& leapPosition) { + float unitScale = 0.001; // convert mm to meters + return _position + _orientation * (leapPosition * unitScale); +} + void Hand::calculateGeometry() { - glm::vec3 offset(0.1, -0.1, -0.15); // place the hand in front of the face where we can see it + glm::vec3 offset(0.2, -0.2, -0.3); // place the hand in front of the face where we can see it Head& head = _owningAvatar->getHead(); _position = head.getPosition() + head.getOrientation() * offset; _orientation = head.getOrientation(); - _numLeapBalls = _fingerPositions.size(); + int numLeapBalls = _fingerTips.size() + _fingerRoots.size(); + _leapBalls.resize(numLeapBalls); - float unitScale = 0.001; // convert mm to meters - for (int b = 0; b < _numLeapBalls; b++) { - glm::vec3 pos = unitScale * _fingerPositions[b] + offset; - _leapBall[b].rotation = _orientation; - _leapBall[b].position = _position + _orientation * pos; + for (int i = 0; i < _fingerTips.size(); ++i) { + _leapBalls[i].rotation = _orientation; + _leapBalls[i].position = leapPositionToWorldPosition(_fingerTips[i]); + _leapBalls[i].radius = 0.01; + _leapBalls[i].touchForce = 0.0; + _leapBalls[i].isCollidable = true; } } @@ -78,23 +82,52 @@ void Hand::render(bool lookingInMirror) { void Hand::renderHandSpheres() { glPushMatrix(); // Draw the leap balls - for (int b = 0; b < _numLeapBalls; b++) { + for (size_t i = 0; i < _leapBalls.size(); i++) { float alpha = 1.0f; if (alpha > 0.0f) { - glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, alpha); // Just to test + glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, alpha); glPushMatrix(); - glTranslatef(_leapBall[b].position.x, _leapBall[b].position.y, _leapBall[b].position.z); - glutSolidSphere(_leapBall[b].radius, 20.0f, 20.0f); + glTranslatef(_leapBalls[i].position.x, _leapBalls[i].position.y, _leapBalls[i].position.z); + glutSolidSphere(_leapBalls[i].radius, 20.0f, 20.0f); glPopMatrix(); } } + + // Draw the finger root cones + if (_fingerTips.size() == _fingerRoots.size()) { + for (size_t i = 0; i < _fingerTips.size(); ++i) { + glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, 0.5); + glm::vec3 tip = leapPositionToWorldPosition(_fingerTips[i]); + glm::vec3 root = leapPositionToWorldPosition(_fingerRoots[i]); + Avatar::renderJointConnectingCone(root, tip, 0.001, 0.003); + } + } + + // Draw the palms + if (_handPositions.size() == _handNormals.size()) { + for (size_t i = 0; i < _handPositions.size(); ++i) { + glColor4f(_ballColor.r, _ballColor.g, _ballColor.b, 0.25); + glm::vec3 tip = leapPositionToWorldPosition(_handPositions[i]); + glm::vec3 root = leapPositionToWorldPosition(_handPositions[i] + (_handNormals[i] * 2.0f)); + Avatar::renderJointConnectingCone(root, tip, 0.05, 0.03); + } + } + glPopMatrix(); } -void Hand::setLeapFingers(const std::vector& fingerPositions) { - _fingerPositions = fingerPositions; +void Hand::setLeapFingers(const std::vector& fingerTips, + const std::vector& fingerRoots) { + _fingerTips = fingerTips; + _fingerRoots = fingerRoots; +} + +void Hand::setLeapHands(const std::vector& handPositions, + const std::vector& handNormals) { + _handPositions = handPositions; + _handNormals = handNormals; } diff --git a/interface/src/Hand.h b/interface/src/Hand.h index 49eed1f817..244ff0dafd 100755 --- a/interface/src/Hand.h +++ b/interface/src/Hand.h @@ -16,6 +16,7 @@ #include "InterfaceConfig.h" #include "SerialInterface.h" #include +#include class Avatar; @@ -41,11 +42,13 @@ public: void render(bool lookingInMirror); void setBallColor (glm::vec3 ballColor ) { _ballColor = ballColor; } - void setLeapFingers (const std::vector& fingerPositions); + void setLeapFingers (const std::vector& fingerTips, + const std::vector& fingerRoots); + void setLeapHands (const std::vector& handPositions, + const std::vector& handNormals); // getters - int getNumLeapBalls () const { return _numLeapBalls;} - const glm::vec3& getLeapBallPosition (int which) const { return _leapBall[which].position;} + const glm::vec3& getLeapBallPosition (int ball) const { return _leapBalls[ball].position;} private: // disallow copies of the Hand, copy of owning Avatar is disallowed too @@ -58,12 +61,12 @@ private: glm::vec3 _ballColor; glm::vec3 _position; glm::quat _orientation; - int _numLeapBalls; - HandBall _leapBall[ MAX_AVATAR_LEAP_BALLS ]; + std::vector _leapBalls; // private methods void renderHandSpheres(); void calculateGeometry(); + glm::vec3 leapPositionToWorldPosition(const glm::vec3& leapPosition); }; #endif diff --git a/interface/src/LeapManager.cpp b/interface/src/LeapManager.cpp index 2c2ce9b8d1..2c5dc5c183 100755 --- a/interface/src/LeapManager.cpp +++ b/interface/src/LeapManager.cpp @@ -11,25 +11,43 @@ #include // needed for RTLD_LAZY #include -bool LeapManager::_isInitialized = false; bool LeapManager::_libraryExists = false; -Leap::Controller* LeapManager::_controller = 0; -HifiLeapListener* LeapManager::_listener = 0; +Leap::Controller* LeapManager::_controller = NULL; +HifiLeapListener* LeapManager::_listener = NULL; class HifiLeapListener : public Leap::Listener { public: Leap::Frame lastFrame; - std::vector fingerPositions; + std::vector fingerTips; + std::vector fingerRoots; + std::vector handPositions; + std::vector handNormals; virtual void onFrame(const Leap::Controller& controller) { #ifndef LEAP_STUBS Leap::Frame frame = controller.frame(); int numFingers = frame.fingers().count(); - fingerPositions.resize(numFingers); + fingerTips.resize(numFingers); + fingerRoots.resize(numFingers); for (int i = 0; i < numFingers; ++i) { const Leap::Finger& thisFinger = frame.fingers()[i]; - const Leap::Vector pos = thisFinger.tipPosition(); - fingerPositions[i] = glm::vec3(pos.x, pos.y, pos.z); + const Leap::Vector pos = thisFinger.stabilizedTipPosition(); + fingerTips[i] = glm::vec3(pos.x, pos.y, pos.z); + + const Leap::Vector root = pos - thisFinger.direction() * thisFinger.length(); + fingerRoots[i] = glm::vec3(root.x, root.y, root.z); + } + + int numHands = frame.hands().count(); + handPositions.resize(numHands); + handNormals.resize(numHands); + for (int i = 0; i < numHands; ++i) { + const Leap::Hand& thisHand = frame.hands()[i]; + const Leap::Vector pos = thisHand.palmPosition(); + handPositions[i] = glm::vec3(pos.x, pos.y, pos.z); + + const Leap::Vector norm = thisHand.palmNormal(); + handNormals[i] = glm::vec3(norm.x, norm.y, norm.z); } lastFrame = frame; #endif @@ -38,28 +56,61 @@ public: }; void LeapManager::initialize() { - if (!_isInitialized) { #ifndef LEAP_STUBS - if (dlopen("/usr/lib/libLeap.dylib", RTLD_LAZY)) { - _libraryExists = true; - _controller = new Leap::Controller(); - _listener = new HifiLeapListener(); - _controller->addListener(*_listener); - } -#endif - _isInitialized = true; + if (dlopen("/usr/lib/libLeap.dylib", RTLD_LAZY)) { + _libraryExists = true; + _controller = new Leap::Controller(); + _listener = new HifiLeapListener(); } +#endif +} + +void LeapManager::terminate() { + delete _listener; + delete _controller; + _listener = NULL; + _controller = NULL; } void LeapManager::nextFrame() { - initialize(); if (_listener && _controller) _listener->onFrame(*_controller); } -const std::vector& LeapManager::getFingerPositions() { - if (_listener) - return _listener->fingerPositions; +const std::vector& LeapManager::getFingerTips() { + if (_listener) { + return _listener->fingerTips; + } + else { + static std::vector empty; + return empty; + } +} + +const std::vector& LeapManager::getFingerRoots() { + if (_listener) { + return _listener->fingerRoots; + } + else { + static std::vector empty; + return empty; + } +} + +const std::vector& LeapManager::getHandPositions() { + if (_listener) { + return _listener->handPositions; + } + else { + static std::vector empty; + return empty; + } +} + +const std::vector& LeapManager::getHandNormals() { + if (_listener) { + return _listener->handNormals; + } else { static std::vector empty; return empty; @@ -69,17 +120,15 @@ const std::vector& LeapManager::getFingerPositions() { std::string LeapManager::statusString() { std::stringstream leapString; #ifndef LEAP_STUBS - if (_isInitialized) { - if (!_libraryExists) - leapString << "Leap library at /usr/lib/libLeap.dylib does not exist."; - else if (!_controller || !_listener || !_controller->devices().count()) - leapString << "Leap controller is not attached."; - else { - leapString << "Leap pointables: " << _listener->lastFrame.pointables().count(); - if (_listener->lastFrame.pointables().count() > 0) { - Leap::Vector pos = _listener->lastFrame.pointables()[0].tipPosition(); - leapString << " pos: " << pos.x << " " << pos.y << " " << pos.z; - } + if (!_libraryExists) + leapString << "Leap library at /usr/lib/libLeap.dylib does not exist."; + else if (!_controller || !_listener || !_controller->devices().count()) + leapString << "Leap controller is not attached."; + else { + leapString << "Leap pointables: " << _listener->lastFrame.pointables().count(); + if (_listener->lastFrame.pointables().count() > 0) { + Leap::Vector pos = _listener->lastFrame.pointables()[0].tipPosition(); + leapString << " pos: " << pos.x << " " << pos.y << " " << pos.z; } } #endif diff --git a/interface/src/LeapManager.h b/interface/src/LeapManager.h index f09bbb8b88..f6ed925c6b 100755 --- a/interface/src/LeapManager.h +++ b/interface/src/LeapManager.h @@ -21,19 +21,18 @@ namespace Leap { class LeapManager { public: static void nextFrame(); // called once per frame to get new Leap data - static const std::vector& getFingerPositions(); + static const std::vector& getFingerTips(); + static const std::vector& getFingerRoots(); + static const std::vector& getHandPositions(); + static const std::vector& getHandNormals(); static std::string statusString(); + static void initialize(); + static void terminate(); private: - static void initialize(); - static bool _isInitialized; // We've looked for the library and hooked it up if it's there. static bool _libraryExists; // The library is present, so we won't crash if we call it. static Leap::Controller* _controller; static HifiLeapListener* _listener; - - - - }; #endif /* defined(__hifi__LeapManager__) */ diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index b09e94bc30..bd3b6a357a 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -125,13 +125,31 @@ int AvatarData::getBroadcastData(unsigned char* destinationBuffer) { *destinationBuffer++ = bitItems; // leap hand data - const std::vector& fingerPositions = _handData->getFingerPositions(); - *destinationBuffer++ = (unsigned char)fingerPositions.size(); - for (size_t i = 0; i < fingerPositions.size(); ++i) - { - destinationBuffer += packFloatScalarToSignedTwoByteFixed(destinationBuffer, fingerPositions[i].x, 4); - destinationBuffer += packFloatScalarToSignedTwoByteFixed(destinationBuffer, fingerPositions[i].y, 4); - destinationBuffer += packFloatScalarToSignedTwoByteFixed(destinationBuffer, fingerPositions[i].z, 4); + // In order to make the hand data version-robust, hand data packing is just a series of vec3's, + // with conventions. If a client doesn't know the conventions, they can just get the vec3's + // and render them as balls, or ignore them, without crashing or disrupting anyone. + // Current convention: + // Zero or more fingetTip positions, followed by the same number of fingerRoot positions + + const std::vector& fingerTips = _handData->getFingerTips(); + const std::vector& fingerRoots = _handData->getFingerRoots(); + size_t numFingerVectors = fingerTips.size() + fingerRoots.size(); + if (numFingerVectors > 255) + numFingerVectors = 0; // safety. We shouldn't ever get over 255, so consider that invalid. + + *destinationBuffer++ = (unsigned char)numFingerVectors; + + if (numFingerVectors > 0) { + for (size_t i = 0; i < fingerTips.size(); ++i) { + destinationBuffer += packFloatScalarToSignedTwoByteFixed(destinationBuffer, fingerTips[i].x, 4); + destinationBuffer += packFloatScalarToSignedTwoByteFixed(destinationBuffer, fingerTips[i].y, 4); + destinationBuffer += packFloatScalarToSignedTwoByteFixed(destinationBuffer, fingerTips[i].z, 4); + } + for (size_t i = 0; i < fingerRoots.size(); ++i) { + destinationBuffer += packFloatScalarToSignedTwoByteFixed(destinationBuffer, fingerRoots[i].x, 4); + destinationBuffer += packFloatScalarToSignedTwoByteFixed(destinationBuffer, fingerRoots[i].y, 4); + destinationBuffer += packFloatScalarToSignedTwoByteFixed(destinationBuffer, fingerRoots[i].z, 4); + } } return destinationBuffer - bufferStart; @@ -229,18 +247,20 @@ int AvatarData::parseData(unsigned char* sourceBuffer, int numBytes) { // leap hand data if (sourceBuffer - startPosition < numBytes) // safety check { - std::vector fingerPositions = _handData->getFingerPositions(); - unsigned int numFingers = *sourceBuffer++; - if (numFingers > MAX_AVATAR_LEAP_BALLS) // safety check - numFingers = 0; - fingerPositions.resize(numFingers); - for (size_t i = 0; i < numFingers; ++i) - { - sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerPositions[i].x), 4); - sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerPositions[i].y), 4); - sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerPositions[i].z), 4); + std::vector fingerTips = _handData->getFingerTips(); + std::vector fingerRoots = _handData->getFingerRoots(); + unsigned int numFingerVectors = *sourceBuffer++; + unsigned int numFingerTips = numFingerVectors / 2; + unsigned int numFingerRoots = numFingerVectors - numFingerTips; + fingerTips.resize(numFingerTips); + fingerRoots.resize(numFingerRoots); + for (size_t i = 0; i < numFingerTips; ++i) { + sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerTips[i].x), 4); + sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerTips[i].y), 4); + sourceBuffer += unpackFloatScalarFromSignedTwoByteFixed((int16_t*) sourceBuffer, &(fingerTips[i].z), 4); } - _handData->setFingerPositions(fingerPositions); + _handData->setFingerTips(fingerTips); + _handData->setFingerRoots(fingerRoots); } return sourceBuffer - startPosition; diff --git a/libraries/avatars/src/HandData.h b/libraries/avatars/src/HandData.h index 86c582a07f..b0d71b4dee 100755 --- a/libraries/avatars/src/HandData.h +++ b/libraries/avatars/src/HandData.h @@ -14,20 +14,27 @@ #include -#define MAX_AVATAR_LEAP_BALLS 10 - class AvatarData; class HandData { public: HandData(AvatarData* owningAvatar); - const std::vector& getFingerPositions() const { return _fingerPositions; } - void setFingerPositions(const std::vector& fingerPositions) { _fingerPositions = fingerPositions; } + const std::vector& getFingerTips() const { return _fingerTips; } + const std::vector& getFingerRoots() const { return _fingerRoots; } + const std::vector& getHandPositions() const { return _handPositions; } + const std::vector& getHandNormals() const { return _handNormals; } + void setFingerTips(const std::vector& fingerTips) { _fingerTips = fingerTips; } + void setFingerRoots(const std::vector& fingerRoots) { _fingerRoots = fingerRoots; } + void setHandPositions(const std::vector& handPositons) { _handPositions = handPositons; } + void setHandNormals(const std::vector& handNormals) { _handNormals = handNormals; } friend class AvatarData; protected: - std::vector _fingerPositions; + std::vector _fingerTips; + std::vector _fingerRoots; + std::vector _handPositions; + std::vector _handNormals; AvatarData* _owningAvatarData; private: // privatize copy ctor and assignment operator so copies of this object cannot be made From cf8257d3e9c25722245a6d93b09b73faf17bdd61 Mon Sep 17 00:00:00 2001 From: Eric Johnston Date: Tue, 2 Jul 2013 17:49:43 -0700 Subject: [PATCH 2/2] fixed curly-braces per request --- interface/src/Hand.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/Hand.cpp b/interface/src/Hand.cpp index ab44464a55..a9732e0233 100755 --- a/interface/src/Hand.cpp +++ b/interface/src/Hand.cpp @@ -29,8 +29,9 @@ Hand::Hand(Avatar* owningAvatar) : void Hand::init() { // Different colors for my hand and others' hands - if (_owningAvatar && _owningAvatar->isMyAvatar()) + if (_owningAvatar && _owningAvatar->isMyAvatar()) { _ballColor = glm::vec3(0.0, 0.4, 0.0); + } else _ballColor = glm::vec3(0.0, 0.0, 0.4); }