diff --git a/cmake/ports/hifi-client-deps/CONTROL b/cmake/ports/hifi-client-deps/CONTROL index 7070cb6fb9..5b9b0bcce0 100644 --- a/cmake/ports/hifi-client-deps/CONTROL +++ b/cmake/ports/hifi-client-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-client-deps Version: 0.1 Description: Collected dependencies for High Fidelity applications -Build-Depends: hifi-deps, aristo (windows), glslang, liblo (windows), nlohmann-json, openvr (windows), quazip (!android), sdl2 (!android), spirv-cross (!android), spirv-tools (!android), sranipal (windows), vulkanmemoryallocator +Build-Depends: hifi-deps, glslang, nlohmann-json, openvr (windows), sdl2 (!android), spirv-cross (!android), spirv-tools (!android), vulkanmemoryallocator, aristo (windows), sranipal (windows) diff --git a/plugins/openvr/CMakeLists.txt b/plugins/openvr/CMakeLists.txt index dcb2e39e1b..e80b2215bf 100644 --- a/plugins/openvr/CMakeLists.txt +++ b/plugins/openvr/CMakeLists.txt @@ -15,5 +15,7 @@ if (WIN32 AND (NOT USE_GLES)) include_hifi_library_headers(octree) target_openvr() + target_sranipal() + target_aristo() target_link_libraries(${TARGET_NAME} Winmm.lib) endif() diff --git a/plugins/openvr/src/ViveControllerManager.cpp b/plugins/openvr/src/ViveControllerManager.cpp index 8aa7311de4..b1d3b791d6 100644 --- a/plugins/openvr/src/ViveControllerManager.cpp +++ b/plugins/openvr/src/ViveControllerManager.cpp @@ -13,6 +13,21 @@ #include #include +#ifdef _WIN32 +#pragma warning( push ) +#pragma warning( disable : 4091 ) +#pragma warning( disable : 4334 ) +#endif + +#include +#include +#include +#include + +#ifdef _WIN32 +#pragma warning( pop ) +#endif + #include #include #include @@ -37,6 +52,8 @@ #include #include +#include "OpenVrDisplayPlugin.h" + extern PoseData _nextSimPoseData; vr::IVRSystem* acquireOpenVrSystem(); @@ -130,6 +147,51 @@ static glm::mat4 calculateResetMat() { return glm::mat4(); } +class ViveProEyeReadThread : public QThread { +public: + ViveProEyeReadThread() { + setObjectName("OpenVR ViveProEye Read Thread"); + } + void run() override { + while (!quit) { + ViveSR::anipal::Eye::EyeData eyeData; + int result = ViveSR::anipal::Eye::GetEyeData(&eyeData); + { + QMutexLocker locker(&eyeDataMutex); + eyeDataBuffer.getEyeDataResult = result; + if (result == ViveSR::Error::WORK) { + uint64_t leftValids = eyeData.verbose_data.left.eye_data_validata_bit_mask; + uint64_t rightValids = eyeData.verbose_data.right.eye_data_validata_bit_mask; + + eyeDataBuffer.leftDirectionValid = + (leftValids & (uint64_t)ViveSR::anipal::Eye::SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY) > (uint64_t)0; + eyeDataBuffer.rightDirectionValid = + (rightValids & (uint64_t)ViveSR::anipal::Eye::SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY) > (uint64_t)0; + eyeDataBuffer.leftOpennessValid = + (leftValids & (uint64_t)ViveSR::anipal::Eye::SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY) > (uint64_t)0; + eyeDataBuffer.rightOpennessValid = + (rightValids & (uint64_t)ViveSR::anipal::Eye::SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY) > (uint64_t)0; + + float *leftGaze = eyeData.verbose_data.left.gaze_direction_normalized.elem_; + float *rightGaze = eyeData.verbose_data.right.gaze_direction_normalized.elem_; + eyeDataBuffer.leftEyeGaze = glm::vec3(leftGaze[0], leftGaze[1], leftGaze[2]); + eyeDataBuffer.rightEyeGaze = glm::vec3(rightGaze[0], rightGaze[1], rightGaze[2]); + + eyeDataBuffer.leftEyeOpenness = eyeData.verbose_data.left.eye_openness; + eyeDataBuffer.rightEyeOpenness = eyeData.verbose_data.right.eye_openness; + } + } + } + } + + bool quit { false }; + + // mutex and buffer for moving data from this thread to the other one + QMutex eyeDataMutex; + EyeDataBuffer eyeDataBuffer; +}; + + static QString outOfRangeDataStrategyToString(ViveControllerManager::OutOfRangeDataStrategy strategy) { switch (strategy) { default: @@ -211,6 +273,81 @@ QString ViveControllerManager::configurationLayout() { return OPENVR_LAYOUT; } +bool isDeviceIndexActive(vr::IVRSystem*& system, uint32_t deviceIndex) { + if (!system) { + return false; + } + if (deviceIndex != vr::k_unTrackedDeviceIndexInvalid && + system->GetTrackedDeviceClass(deviceIndex) == vr::TrackedDeviceClass_Controller && + system->IsTrackedDeviceConnected(deviceIndex)) { + vr::EDeviceActivityLevel activityLevel = system->GetTrackedDeviceActivityLevel(deviceIndex); + if (activityLevel == vr::k_EDeviceActivityLevel_UserInteraction) { + return true; + } + } + return false; +} + +bool isHandControllerActive(vr::IVRSystem*& system, vr::ETrackedControllerRole deviceRole) { + if (!system) { + return false; + } + auto deviceIndex = system->GetTrackedDeviceIndexForControllerRole(deviceRole); + return isDeviceIndexActive(system, deviceIndex); +} + +bool areBothHandControllersActive(vr::IVRSystem*& system) { + return + isHandControllerActive(system, vr::TrackedControllerRole_LeftHand) && + isHandControllerActive(system, vr::TrackedControllerRole_RightHand); +} + + +void ViveControllerManager::enableGestureDetection() { + if (_viveCameraHandTracker) { + return; + } + if (!ViveSR::anipal::Eye::IsViveProEye()) { + return; + } + +// #define HAND_TRACKER_USE_EXTERNAL_TRANSFORM 1 + +#ifdef HAND_TRACKER_USE_EXTERNAL_TRANSFORM + UseExternalTransform(true); // camera hand tracker results are in HMD frame +#else + UseExternalTransform(false); // camera hand tracker results are in sensor frame +#endif + GestureOption options; // defaults are GestureBackendAuto and GestureModeSkeleton + GestureFailure gestureFailure = StartGestureDetection(&options); + switch (gestureFailure) { + case GestureFailureNone: + qDebug() << "StartGestureDetection success"; + _viveCameraHandTracker = true; + break; + case GestureFailureOpenCL: + qDebug() << "StartGestureDetection (Only on Windows) OpenCL is not supported on the machine"; + break; + case GestureFailureCamera: + qDebug() << "StartGestureDetection Start camera failed"; + break; + case GestureFailureInternal: + qDebug() << "StartGestureDetection Internal errors"; + break; + case GestureFailureCPUOnPC: + qDebug() << "StartGestureDetection CPU backend is not supported on Windows"; + break; + } +} + +void ViveControllerManager::disableGestureDetection() { + if (!_viveCameraHandTracker) { + return; + } + StopGestureDetection(); + _viveCameraHandTracker = false; +} + bool ViveControllerManager::activate() { InputPlugin::activate(); @@ -230,6 +367,28 @@ bool ViveControllerManager::activate() { auto userInputMapper = DependencyManager::get(); userInputMapper->registerDevice(_inputDevice); _registeredWithInputMapper = true; + + if (ViveSR::anipal::Eye::IsViveProEye()) { + qDebug() << "Vive Pro eye-tracking detected"; + + int error = ViveSR::anipal::Initial(ViveSR::anipal::Eye::ANIPAL_TYPE_EYE, NULL); + if (error == ViveSR::Error::WORK) { + _viveProEye = true; + qDebug() << "Successfully initialize Eye engine."; + } else if (error == ViveSR::Error::RUNTIME_NOT_FOUND) { + _viveProEye = false; + qDebug() << "please follows SRanipal SDK guide to install SR_Runtime first"; + } else { + _viveProEye = false; + qDebug() << "Failed to initialize Eye engine. please refer to ViveSR error code:" << error; + } + + if (_viveProEye) { + _viveProEyeReadThread = std::make_shared(); + _viveProEyeReadThread->start(QThread::HighPriority); + } + } + return true; } @@ -251,6 +410,13 @@ void ViveControllerManager::deactivate() { userInputMapper->removeDevice(_inputDevice->_deviceID); _registeredWithInputMapper = false; + if (_viveProEyeReadThread) { + _viveProEyeReadThread->quit = true; + _viveProEyeReadThread->wait(); + _viveProEyeReadThread = nullptr; + ViveSR::anipal::Release(ViveSR::anipal::Eye::ANIPAL_TYPE_EYE); + } + saveSettings(); } @@ -262,6 +428,311 @@ bool ViveControllerManager::isHeadControllerMounted() const { return activityLevel == vr::k_EDeviceActivityLevel_UserInteraction; } +void ViveControllerManager::invalidateEyeInputs() { + _inputDevice->_poseStateMap[controller::LEFT_EYE].valid = false; + _inputDevice->_poseStateMap[controller::RIGHT_EYE].valid = false; + _inputDevice->_axisStateMap[controller::LEFT_EYE_BLINK].valid = false; + _inputDevice->_axisStateMap[controller::RIGHT_EYE_BLINK].valid = false; +} + + +void ViveControllerManager::updateEyeTracker(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) { + if (!isHeadControllerMounted()) { + invalidateEyeInputs(); + return; + } + + EyeDataBuffer eyeDataBuffer; + { + // GetEyeData takes around 4ms to finish, so we run it on a thread. + QMutexLocker locker(&_viveProEyeReadThread->eyeDataMutex); + memcpy(&eyeDataBuffer, &_viveProEyeReadThread->eyeDataBuffer, sizeof(eyeDataBuffer)); + } + + if (eyeDataBuffer.getEyeDataResult != ViveSR::Error::WORK) { + invalidateEyeInputs(); + return; + } + + // only update from buffer values if the new data is "valid" + if (!eyeDataBuffer.leftDirectionValid) { + eyeDataBuffer.leftEyeGaze = _prevEyeData.leftEyeGaze; + eyeDataBuffer.leftDirectionValid = _prevEyeData.leftDirectionValid; + } + if (!eyeDataBuffer.rightDirectionValid) { + eyeDataBuffer.rightEyeGaze = _prevEyeData.rightEyeGaze; + eyeDataBuffer.rightDirectionValid = _prevEyeData.rightDirectionValid; + } + if (!eyeDataBuffer.leftOpennessValid) { + eyeDataBuffer.leftEyeOpenness = _prevEyeData.leftEyeOpenness; + eyeDataBuffer.leftOpennessValid = _prevEyeData.leftOpennessValid; + } + if (!eyeDataBuffer.rightOpennessValid) { + eyeDataBuffer.rightEyeOpenness = _prevEyeData.rightEyeOpenness; + eyeDataBuffer.rightOpennessValid = _prevEyeData.rightOpennessValid; + } + _prevEyeData = eyeDataBuffer; + + // transform data into what the controller system expects. + + // in the data from sranipal, left=+x, up=+y, forward=+z + mat4 localLeftEyeMat = glm::lookAt(vec3(0.0f, 0.0f, 0.0f), + glm::vec3(-eyeDataBuffer.leftEyeGaze[0], + eyeDataBuffer.leftEyeGaze[1], + eyeDataBuffer.leftEyeGaze[2]), + vec3(0.0f, 1.0f, 0.0f)); + quat localLeftEyeRot = glm::quat_cast(localLeftEyeMat); + quat avatarLeftEyeRot = _inputDevice->_poseStateMap[controller::HEAD].rotation * localLeftEyeRot; + + mat4 localRightEyeMat = glm::lookAt(vec3(0.0f, 0.0f, 0.0f), + glm::vec3(-eyeDataBuffer.rightEyeGaze[0], + eyeDataBuffer.rightEyeGaze[1], + eyeDataBuffer.rightEyeGaze[2]), + vec3(0.0f, 1.0f, 0.0f)); + quat localRightEyeRot = glm::quat_cast(localRightEyeMat); + quat avatarRightEyeRot = _inputDevice->_poseStateMap[controller::HEAD].rotation * localRightEyeRot; + + // TODO -- figure out translations for eyes + if (eyeDataBuffer.leftDirectionValid) { + _inputDevice->_poseStateMap[controller::LEFT_EYE] = controller::Pose(glm::vec3(), avatarLeftEyeRot); + _inputDevice->_poseStateMap[controller::LEFT_EYE].valid = true; + } else { + _inputDevice->_poseStateMap[controller::LEFT_EYE].valid = false; + } + if (eyeDataBuffer.rightDirectionValid) { + _inputDevice->_poseStateMap[controller::RIGHT_EYE] = controller::Pose(glm::vec3(), avatarRightEyeRot); + _inputDevice->_poseStateMap[controller::RIGHT_EYE].valid = true; + } else { + _inputDevice->_poseStateMap[controller::RIGHT_EYE].valid = false; + } + + quint64 now = usecTimestampNow(); + + // in hifi, 0 is open 1 is closed. in SRanipal 1 is open, 0 is closed. + if (eyeDataBuffer.leftOpennessValid) { + _inputDevice->_axisStateMap[controller::LEFT_EYE_BLINK] = + controller::AxisValue(1.0f - eyeDataBuffer.leftEyeOpenness, now); + } else { + _inputDevice->_poseStateMap[controller::LEFT_EYE_BLINK].valid = false; + } + if (eyeDataBuffer.rightOpennessValid) { + _inputDevice->_axisStateMap[controller::RIGHT_EYE_BLINK] = + controller::AxisValue(1.0f - eyeDataBuffer.rightEyeOpenness, now); + } else { + _inputDevice->_poseStateMap[controller::RIGHT_EYE_BLINK].valid = false; + } +} + +glm::vec3 ViveControllerManager::getRollingAverageHandPoint(int handIndex, int pointIndex) const { +#if 0 + return _handPoints[0][handIndex][pointIndex]; +#else + glm::vec3 result; + for (int s = 0; s < NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES; s++) { + result += _handPoints[s][handIndex][pointIndex]; + } + return result / NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES; +#endif +} + + +controller::Pose ViveControllerManager::trackedHandDataToPose(int hand, const glm::vec3& palmFacing, + int nearHandPositionIndex, int farHandPositionIndex) { + glm::vec3 nearPoint = getRollingAverageHandPoint(hand, nearHandPositionIndex); + + glm::quat poseRot; + if (nearHandPositionIndex != farHandPositionIndex) { + glm::vec3 farPoint = getRollingAverageHandPoint(hand, farHandPositionIndex); + + glm::vec3 pointingDir = farPoint - nearPoint; // y axis + glm::vec3 otherAxis = glm::cross(pointingDir, palmFacing); + + glm::mat4 rotMat; + rotMat = glm::mat4(glm::vec4(otherAxis, 0.0f), + glm::vec4(pointingDir, 0.0f), + glm::vec4(palmFacing * (hand == 0 ? 1.0f : -1.0f), 0.0f), + glm::vec4(0.0f, 0.0f, 0.0f, 1.0f)); + poseRot = glm::normalize(glmExtractRotation(rotMat)); + } + + if (!isNaN(poseRot)) { + controller::Pose pose(nearPoint, poseRot); + return pose; + } else { + controller::Pose pose; + pose.valid = false; + return pose; + } +} + + +void ViveControllerManager::trackFinger(int hand, int jointIndex1, int jointIndex2, int jointIndex3, int jointIndex4, + controller::StandardPoseChannel joint1, controller::StandardPoseChannel joint2, + controller::StandardPoseChannel joint3, controller::StandardPoseChannel joint4) { + + glm::vec3 point1 = getRollingAverageHandPoint(hand, jointIndex1); + glm::vec3 point2 = getRollingAverageHandPoint(hand, jointIndex2); + glm::vec3 point3 = getRollingAverageHandPoint(hand, jointIndex3); + glm::vec3 point4 = getRollingAverageHandPoint(hand, jointIndex4); + + glm::vec3 wristPos = getRollingAverageHandPoint(hand, 0); + glm::vec3 thumb2 = getRollingAverageHandPoint(hand, 2); + glm::vec3 pinkie1 = getRollingAverageHandPoint(hand, 17); + + // 1st + glm::vec3 palmFacing = glm::normalize(glm::cross(pinkie1 - wristPos, thumb2 - wristPos)); + glm::vec3 handForward = glm::normalize(point1 - wristPos); + glm::vec3 x = glm::normalize(glm::cross(palmFacing, handForward)); + glm::vec3 y = glm::normalize(point2 - point1); + glm::vec3 z = (hand == 0) ? glm::cross(y, x) : glm::cross(x, y); + glm::mat4 rotMat1 = glm::mat4(glm::vec4(x, 0.0f), + glm::vec4(y, 0.0f), + glm::vec4(z, 0.0f), + glm::vec4(0.0f, 0.0f, 0.0f, 1.0f)); + glm::quat rot1 = glm::normalize(glmExtractRotation(rotMat1)); + if (!isNaN(rot1)) { + _inputDevice->_poseStateMap[joint1] = controller::Pose(point1, rot1); + } + + + // 2nd + glm::vec3 x2 = x; // glm::normalize(glm::cross(point3 - point2, point2 - point1)); + glm::vec3 y2 = glm::normalize(point3 - point2); + glm::vec3 z2 = (hand == 0) ? glm::cross(y2, x2) : glm::cross(x2, y2); + + glm::mat4 rotMat2 = glm::mat4(glm::vec4(x2, 0.0f), + glm::vec4(y2, 0.0f), + glm::vec4(z2, 0.0f), + glm::vec4(0.0f, 0.0f, 0.0f, 1.0f)); + glm::quat rot2 = glm::normalize(glmExtractRotation(rotMat2)); + if (!isNaN(rot2)) { + _inputDevice->_poseStateMap[joint2] = controller::Pose(point2, rot2); + } + + + // 3rd + glm::vec3 x3 = x; // glm::normalize(glm::cross(point4 - point3, point3 - point1)); + glm::vec3 y3 = glm::normalize(point4 - point3); + glm::vec3 z3 = (hand == 0) ? glm::cross(y3, x3) : glm::cross(x3, y3); + + glm::mat4 rotMat3 = glm::mat4(glm::vec4(x3, 0.0f), + glm::vec4(y3, 0.0f), + glm::vec4(z3, 0.0f), + glm::vec4(0.0f, 0.0f, 0.0f, 1.0f)); + glm::quat rot3 = glm::normalize(glmExtractRotation(rotMat3)); + if (!isNaN(rot3)) { + _inputDevice->_poseStateMap[joint3] = controller::Pose(point3, rot3); + } + + + // 4th + glm::quat rot4 = rot3; + if (!isNaN(rot4)) { + _inputDevice->_poseStateMap[joint4] = controller::Pose(point4, rot4); + } +} + + +void ViveControllerManager::updateCameraHandTracker(float deltaTime, + const controller::InputCalibrationData& inputCalibrationData) { + + if (areBothHandControllersActive(_system)) { + // if both hand-controllers are in use, don't do camera hand tracking + disableGestureDetection(); + } else { + enableGestureDetection(); + } + + if (!_viveCameraHandTracker) { + return; + } + + const GestureResult* results = NULL; + int handTrackerFrameIndex { -1 }; + int resultsHandCount = GetGestureResult(&results, &handTrackerFrameIndex); + + if (handTrackerFrameIndex >= 0 /* && handTrackerFrameIndex != _lastHandTrackerFrameIndex */) { +#ifdef HAND_TRACKER_USE_EXTERNAL_TRANSFORM + glm::mat4 trackedHandToAvatar = + glm::inverse(inputCalibrationData.avatarMat) * + inputCalibrationData.sensorToWorldMat * + inputCalibrationData.hmdSensorMat; + // glm::mat4 trackedHandToAvatar = _inputDevice->_poseStateMap[controller::HEAD].getMatrix() * Matrices::Y_180; +#else + DisplayPluginPointer displayPlugin = _container->getActiveDisplayPlugin(); + std::shared_ptr openVRDisplayPlugin = + std::dynamic_pointer_cast(displayPlugin); + glm::mat4 sensorResetMatrix; + if (openVRDisplayPlugin) { + sensorResetMatrix = openVRDisplayPlugin->getSensorResetMatrix(); + } + + glm::mat4 trackedHandToAvatar = + glm::inverse(inputCalibrationData.avatarMat) * + inputCalibrationData.sensorToWorldMat * + sensorResetMatrix; +#endif + + // roll all the old points in the rolling average + memmove(&(_handPoints[1]), + &(_handPoints[0]), + sizeof(_handPoints[0]) * (NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES - 1)); + + for (int handIndex = 0; handIndex < resultsHandCount; handIndex++) { + bool isLeftHand = results[handIndex].isLeft; + + vr::ETrackedControllerRole controllerRole = + isLeftHand ? vr::TrackedControllerRole_LeftHand : vr::TrackedControllerRole_RightHand; + if (isHandControllerActive(_system, controllerRole)) { + continue; // if the controller for this hand is tracked, ignore camera hand tracking + } + + int hand = isLeftHand ? 0 : 1; + for (int pointIndex = 0; pointIndex < NUMBER_OF_HAND_POINTS; pointIndex++) { + glm::vec3 pos(results[handIndex].points[3 * pointIndex], + results[handIndex].points[3 * pointIndex + 1], + -results[handIndex].points[3 * pointIndex + 2]); + _handPoints[0][hand][pointIndex] = transformPoint(trackedHandToAvatar, pos); + } + + glm::vec3 wristPos = getRollingAverageHandPoint(hand, 0); + glm::vec3 thumb2 = getRollingAverageHandPoint(hand, 2); + glm::vec3 pinkie1 = getRollingAverageHandPoint(hand, 17); + glm::vec3 palmFacing = glm::cross(pinkie1 - wristPos, thumb2 - wristPos); // z axis + + _inputDevice->_poseStateMap[isLeftHand ? controller::LEFT_HAND : controller::RIGHT_HAND] = + trackedHandDataToPose(hand, palmFacing, 0, 9); + trackFinger(hand, 1, 2, 3, 4, + isLeftHand ? controller::LEFT_HAND_THUMB1 : controller::RIGHT_HAND_THUMB1, + isLeftHand ? controller::LEFT_HAND_THUMB2 : controller::RIGHT_HAND_THUMB2, + isLeftHand ? controller::LEFT_HAND_THUMB3 : controller::RIGHT_HAND_THUMB3, + isLeftHand ? controller::LEFT_HAND_THUMB4 : controller::RIGHT_HAND_THUMB4); + trackFinger(hand, 5, 6, 7, 8, + isLeftHand ? controller::LEFT_HAND_INDEX1 : controller::RIGHT_HAND_INDEX1, + isLeftHand ? controller::LEFT_HAND_INDEX2 : controller::RIGHT_HAND_INDEX2, + isLeftHand ? controller::LEFT_HAND_INDEX3 : controller::RIGHT_HAND_INDEX3, + isLeftHand ? controller::LEFT_HAND_INDEX4 : controller::RIGHT_HAND_INDEX4); + trackFinger(hand, 9, 10, 11, 12, + isLeftHand ? controller::LEFT_HAND_MIDDLE1 : controller::RIGHT_HAND_MIDDLE1, + isLeftHand ? controller::LEFT_HAND_MIDDLE2 : controller::RIGHT_HAND_MIDDLE2, + isLeftHand ? controller::LEFT_HAND_MIDDLE3 : controller::RIGHT_HAND_MIDDLE3, + isLeftHand ? controller::LEFT_HAND_MIDDLE4 : controller::RIGHT_HAND_MIDDLE4); + trackFinger(hand, 13, 14, 15, 16, + isLeftHand ? controller::LEFT_HAND_RING1 : controller::RIGHT_HAND_RING1, + isLeftHand ? controller::LEFT_HAND_RING2 : controller::RIGHT_HAND_RING2, + isLeftHand ? controller::LEFT_HAND_RING3 : controller::RIGHT_HAND_RING3, + isLeftHand ? controller::LEFT_HAND_RING4 : controller::RIGHT_HAND_RING4); + trackFinger(hand, 17, 18, 19, 20, + isLeftHand ? controller::LEFT_HAND_PINKY1 : controller::RIGHT_HAND_PINKY1, + isLeftHand ? controller::LEFT_HAND_PINKY2 : controller::RIGHT_HAND_PINKY2, + isLeftHand ? controller::LEFT_HAND_PINKY3 : controller::RIGHT_HAND_PINKY3, + isLeftHand ? controller::LEFT_HAND_PINKY4 : controller::RIGHT_HAND_PINKY4); + } + } + _lastHandTrackerFrameIndex = handTrackerFrameIndex; +} + + void ViveControllerManager::pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) { if (!_system) { @@ -297,6 +768,12 @@ void ViveControllerManager::pluginUpdate(float deltaTime, const controller::Inpu userInputMapper->registerDevice(_inputDevice); _registeredWithInputMapper = true; } + + if (_viveProEye) { + updateEyeTracker(deltaTime, inputCalibrationData); + } + + updateCameraHandTracker(deltaTime, inputCalibrationData); } void ViveControllerManager::loadSettings() { @@ -830,9 +1307,7 @@ void ViveControllerManager::InputDevice::handleHmd(uint32_t deviceIndex, const c void ViveControllerManager::InputDevice::handleHandController(float deltaTime, uint32_t deviceIndex, const controller::InputCalibrationData& inputCalibrationData, bool isLeftHand) { - if (_system->IsTrackedDeviceConnected(deviceIndex) && - _system->GetTrackedDeviceClass(deviceIndex) == vr::TrackedDeviceClass_Controller && - _nextSimPoseData.vrPoses[deviceIndex].bPoseIsValid) { + if (isDeviceIndexActive(_system, deviceIndex) && _nextSimPoseData.vrPoses[deviceIndex].bPoseIsValid) { // process pose const mat4& mat = _nextSimPoseData.poses[deviceIndex]; @@ -1401,9 +1876,52 @@ controller::Input::NamedVector ViveControllerManager::InputDevice::getAvailableI makePair(LEFT_GRIP, "LeftGrip"), makePair(RIGHT_GRIP, "RightGrip"), - // 3d location of controller + // 3d location of left controller and fingers makePair(LEFT_HAND, "LeftHand"), + makePair(LEFT_HAND_THUMB1, "LeftHandThumb1"), + makePair(LEFT_HAND_THUMB2, "LeftHandThumb2"), + makePair(LEFT_HAND_THUMB3, "LeftHandThumb3"), + makePair(LEFT_HAND_THUMB4, "LeftHandThumb4"), + makePair(LEFT_HAND_INDEX1, "LeftHandIndex1"), + makePair(LEFT_HAND_INDEX2, "LeftHandIndex2"), + makePair(LEFT_HAND_INDEX3, "LeftHandIndex3"), + makePair(LEFT_HAND_INDEX4, "LeftHandIndex4"), + makePair(LEFT_HAND_MIDDLE1, "LeftHandMiddle1"), + makePair(LEFT_HAND_MIDDLE2, "LeftHandMiddle2"), + makePair(LEFT_HAND_MIDDLE3, "LeftHandMiddle3"), + makePair(LEFT_HAND_MIDDLE4, "LeftHandMiddle4"), + makePair(LEFT_HAND_RING1, "LeftHandRing1"), + makePair(LEFT_HAND_RING2, "LeftHandRing2"), + makePair(LEFT_HAND_RING3, "LeftHandRing3"), + makePair(LEFT_HAND_RING4, "LeftHandRing4"), + makePair(LEFT_HAND_PINKY1, "LeftHandPinky1"), + makePair(LEFT_HAND_PINKY2, "LeftHandPinky2"), + makePair(LEFT_HAND_PINKY3, "LeftHandPinky3"), + makePair(LEFT_HAND_PINKY4, "LeftHandPinky4"), + + // 3d location of right controller and fingers makePair(RIGHT_HAND, "RightHand"), + makePair(RIGHT_HAND_THUMB1, "RightHandThumb1"), + makePair(RIGHT_HAND_THUMB2, "RightHandThumb2"), + makePair(RIGHT_HAND_THUMB3, "RightHandThumb3"), + makePair(RIGHT_HAND_THUMB4, "RightHandThumb4"), + makePair(RIGHT_HAND_INDEX1, "RightHandIndex1"), + makePair(RIGHT_HAND_INDEX2, "RightHandIndex2"), + makePair(RIGHT_HAND_INDEX3, "RightHandIndex3"), + makePair(RIGHT_HAND_INDEX4, "RightHandIndex4"), + makePair(RIGHT_HAND_MIDDLE1, "RightHandMiddle1"), + makePair(RIGHT_HAND_MIDDLE2, "RightHandMiddle2"), + makePair(RIGHT_HAND_MIDDLE3, "RightHandMiddle3"), + makePair(RIGHT_HAND_MIDDLE4, "RightHandMiddle4"), + makePair(RIGHT_HAND_RING1, "RightHandRing1"), + makePair(RIGHT_HAND_RING2, "RightHandRing2"), + makePair(RIGHT_HAND_RING3, "RightHandRing3"), + makePair(RIGHT_HAND_RING4, "RightHandRing4"), + makePair(RIGHT_HAND_PINKY1, "RightHandPinky1"), + makePair(RIGHT_HAND_PINKY2, "RightHandPinky2"), + makePair(RIGHT_HAND_PINKY3, "RightHandPinky3"), + makePair(RIGHT_HAND_PINKY4, "RightHandPinky4"), + makePair(LEFT_FOOT, "LeftFoot"), makePair(RIGHT_FOOT, "RightFoot"), makePair(HIPS, "Hips"), @@ -1411,6 +1929,10 @@ controller::Input::NamedVector ViveControllerManager::InputDevice::getAvailableI makePair(HEAD, "Head"), makePair(LEFT_ARM, "LeftArm"), makePair(RIGHT_ARM, "RightArm"), + makePair(LEFT_EYE, "LeftEye"), + makePair(RIGHT_EYE, "RightEye"), + makePair(LEFT_EYE_BLINK, "LeftEyeBlink"), + makePair(RIGHT_EYE_BLINK, "RightEyeBlink"), // 16 tracked poses makePair(TRACKED_OBJECT_00, "TrackedObject00"), diff --git a/plugins/openvr/src/ViveControllerManager.h b/plugins/openvr/src/ViveControllerManager.h index dbd248dc53..66462cbe85 100644 --- a/plugins/openvr/src/ViveControllerManager.h +++ b/plugins/openvr/src/ViveControllerManager.h @@ -31,6 +31,23 @@ namespace vr { class IVRSystem; } +class ViveProEyeReadThread; + +class EyeDataBuffer { +public: + int getEyeDataResult { 0 }; + bool leftDirectionValid { false }; + bool rightDirectionValid { false }; + bool leftOpennessValid { false }; + bool rightOpennessValid { false }; + glm::vec3 leftEyeGaze; + glm::vec3 rightEyeGaze; + float leftEyeOpenness { 0.0f }; + float rightEyeOpenness { 0.0f }; +}; + + + class ViveControllerManager : public InputPlugin { Q_OBJECT public: @@ -49,12 +66,18 @@ public: bool isHeadController() const override { return true; } bool isHeadControllerMounted() const; + void enableGestureDetection(); + void disableGestureDetection(); + bool activate() override; void deactivate() override; QString getDeviceName() { return QString::fromStdString(_inputDevice->_headsetName); } void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); } + void invalidateEyeInputs(); + void updateEyeTracker(float deltaTime, const controller::InputCalibrationData& inputCalibrationData); + void updateCameraHandTracker(float deltaTime, const controller::InputCalibrationData& inputCalibrationData); void pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override; virtual void saveSettings() const override; @@ -229,6 +252,24 @@ private: vr::IVRSystem* _system { nullptr }; std::shared_ptr _inputDevice { std::make_shared(_system) }; + bool _viveProEye { false }; + mutable std::recursive_mutex _getEyeDataLock; + std::shared_ptr _viveProEyeReadThread; + EyeDataBuffer _prevEyeData; + + bool _viveCameraHandTracker { false }; + int _lastHandTrackerFrameIndex { -1 }; + + const static int NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES { 6 }; + const static int NUMBER_OF_HAND_POINTS { 21 }; + glm::vec3 _handPoints[NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES][2][NUMBER_OF_HAND_POINTS]; // 2 for number of hands + glm::vec3 getRollingAverageHandPoint(int handIndex, int pointIndex) const; + controller::Pose trackedHandDataToPose(int hand, const glm::vec3& palmFacing, + int nearHandPositionIndex, int farHandPositionIndex); + void trackFinger(int hand, int jointIndex1, int jointIndex2, int jointIndex3, int jointIndex4, + controller::StandardPoseChannel joint1, controller::StandardPoseChannel joint2, + controller::StandardPoseChannel joint3, controller::StandardPoseChannel joint4); + static const char* NAME; };