From 473bd4d64e94ad84f54dbeb3b2dcadccb6c56734 Mon Sep 17 00:00:00 2001 From: Lubosz Sarnecki Date: Mon, 12 Feb 2024 15:55:31 +0100 Subject: [PATCH] plugins: Add OpenXR plugin. Add OpenXrDisplayPlugin and OpenXrInputPlugin. Add controller bindings for the Valve Index controller. --- .../resources/controllers/openxr_index.json | 45 ++ plugins/CMakeLists.txt | 3 + plugins/openxr/CMakeLists.txt | 25 + plugins/openxr/src/OpenXrContext.cpp | 387 ++++++++++++ plugins/openxr/src/OpenXrContext.h | 82 +++ plugins/openxr/src/OpenXrDisplayPlugin.cpp | 551 ++++++++++++++++++ plugins/openxr/src/OpenXrDisplayPlugin.h | 97 +++ plugins/openxr/src/OpenXrInputPlugin.cpp | 523 +++++++++++++++++ plugins/openxr/src/OpenXrInputPlugin.h | 104 ++++ plugins/openxr/src/OpenXrProvider.cpp | 59 ++ plugins/openxr/src/plugin.json | 4 + 11 files changed, 1880 insertions(+) create mode 100644 interface/resources/controllers/openxr_index.json create mode 100644 plugins/openxr/CMakeLists.txt create mode 100644 plugins/openxr/src/OpenXrContext.cpp create mode 100644 plugins/openxr/src/OpenXrContext.h create mode 100644 plugins/openxr/src/OpenXrDisplayPlugin.cpp create mode 100644 plugins/openxr/src/OpenXrDisplayPlugin.h create mode 100644 plugins/openxr/src/OpenXrInputPlugin.cpp create mode 100644 plugins/openxr/src/OpenXrInputPlugin.h create mode 100644 plugins/openxr/src/OpenXrProvider.cpp create mode 100644 plugins/openxr/src/plugin.json diff --git a/interface/resources/controllers/openxr_index.json b/interface/resources/controllers/openxr_index.json new file mode 100644 index 0000000000..774b55981e --- /dev/null +++ b/interface/resources/controllers/openxr_index.json @@ -0,0 +1,45 @@ +{ + "name": "OpenXR Index to Standard", + "channels": [ + { "from": "Index.LeftHand", "to": "Standard.LeftHand" }, + { "from": "Index.RightHand", "to": "Standard.RightHand" }, + { "from": "Index.Head", "to" : "Standard.Head", "when" : [ "Application.InHMD"] }, + + { "from": "Index.A", "to": "Standard.RightPrimaryThumb", "peek": true }, + { "from": "Index.B", "to": "Standard.RightSecondaryThumb", "peek": true }, + { "from": "Index.X", "to": "Standard.LeftPrimaryThumb", "peek": true }, + { "from": "Index.Y", "to": "Standard.LeftSecondaryThumb", "peek": true}, + + { "from": "Index.A", "to": "Standard.A" }, + { "from": "Index.B", "to": "Standard.B" }, + { "from": "Index.X", "to": "Standard.X" }, + { "from": "Index.Y", "to": "Standard.Y" }, + + { "from": "Index.LeftPrimaryThumbTouch", "to": "Standard.LeftPrimaryThumbTouch" }, + { "from": "Index.LeftSecondaryThumbTouch", "to": "Standard.LeftSecondaryThumbTouch" }, + { "from": "Index.LeftThumbUp", "to": "Standard.LeftThumbUp" }, + + { "from": "Index.RightPrimaryThumbTouch", "to": "Standard.RightPrimaryThumbTouch" }, + { "from": "Index.RightSecondaryThumbTouch", "to": "Standard.RightSecondaryThumbTouch" }, + { "from": "Index.RightThumbUp", "to": "Standard.RightThumbUp" }, + + { "from": "Index.LY", "to": "Standard.LY" }, + { "from": "Index.LX", "to": "Standard.LX" }, + { "from": "Index.RY", "to": "Standard.RY" }, + { "from": "Index.RX", "to": "Standard.RX" }, + { "from": "Index.LSTouch", "to": "Standard.LSTouch" }, + { "from": "Index.RSTouch", "to": "Standard.RSTouch" }, + + { "from": "Index.RT", "to": "Standard.RT" }, + { "from": "Index.LT", "to": "Standard.LT" }, + { "from": "Index.RTClick", "to": "Standard.RTClick" }, + { "from": "Index.LTClick", "to": "Standard.LTClick" }, + { "from": "Index.LeftPrimaryIndexTouch", "to": "Standard.LeftPrimaryIndexTouch" }, + { "from": "Index.RightPrimaryIndexTouch", "to": "Standard.RightPrimaryIndexTouch" }, + { "from": "Index.LeftIndexPoint", "to": "Standard.LeftIndexPoint" }, + { "from": "Index.RightIndexPoint", "to": "Standard.RightIndexPoint" }, + + { "from": "Index.LeftApplicationMenu", "to": "Standard.Back" }, + { "from": "Index.RightApplicationMenu", "to": "Standard.Start" } + ] +} diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index c68abefa77..5e22dfc41d 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -26,6 +26,9 @@ if (NOT SERVER_ONLY AND NOT ANDROID) add_subdirectory(${DIR}) endif() + set(DIR "openxr") + add_subdirectory(${DIR}) + set(DIR "hifiSdl2") add_subdirectory(${DIR}) diff --git a/plugins/openxr/CMakeLists.txt b/plugins/openxr/CMakeLists.txt new file mode 100644 index 0000000000..2e7eb9f821 --- /dev/null +++ b/plugins/openxr/CMakeLists.txt @@ -0,0 +1,25 @@ +# +# Copyright 2024 Lubosz Sarnecki +# +# SPDX-License-Identifier: Apache-2.0 +# + +find_package(OpenXR REQUIRED) +if (NOT OpenXR_FOUND) + MESSAGE(FATAL_ERROR "OpenXR not found!") +endif() + +set(TARGET_NAME openxr) +setup_hifi_plugin(Gui Qml Multimedia) +link_hifi_libraries(shared task gl qml networking controllers ui + plugins display-plugins ui-plugins input-plugins + audio-client render-utils graphics shaders gpu render + material-networking model-networking model-baker hfm + model-serializers ktx image procedural ${PLATFORM_GL_BACKEND} OpenXR::openxr_loader) +include_hifi_library_headers(octree) +include_hifi_library_headers(script-engine) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + # Silence GCC warnings + target_compile_options(openxr PRIVATE -Wno-missing-field-initializers) +endif() diff --git a/plugins/openxr/src/OpenXrContext.cpp b/plugins/openxr/src/OpenXrContext.cpp new file mode 100644 index 0000000000..700d488747 --- /dev/null +++ b/plugins/openxr/src/OpenXrContext.cpp @@ -0,0 +1,387 @@ +// +// Overte OpenXR Plugin +// +// Copyright 2024 Lubosz Sarnecki +// +// SPDX-License-Identifier: Apache-2.0 +// + +#include "OpenXrContext.h" +#include + +#include + +#include + +#define XR_USE_PLATFORM_XLIB +#define XR_USE_GRAPHICS_API_OPENGL +#include +#include + +Q_DECLARE_LOGGING_CATEGORY(xr_context_cat) +Q_LOGGING_CATEGORY(xr_context_cat, "openxr.context") + +// Checks XrResult, returns false on errors and logs the error as qCritical. +bool xrCheck(XrInstance instance, XrResult result, const char* message) { + if (XR_SUCCEEDED(result)) + return true; + + char errorName[XR_MAX_RESULT_STRING_SIZE]; + if (instance != XR_NULL_HANDLE) { + xrResultToString(instance, result, errorName); + } else { + sprintf(errorName, "%d", result); + } + + qCCritical(xr_context_cat, "%s: %s", errorName, message); + + return false; +} + +// Extension functions must be loaded with xrGetInstanceProcAddr +static PFN_xrGetOpenGLGraphicsRequirementsKHR pfnGetOpenGLGraphicsRequirementsKHR = nullptr; +static bool initFunctionPointers(XrInstance instance) { + XrResult result = xrGetInstanceProcAddr(instance, "xrGetOpenGLGraphicsRequirementsKHR", + (PFN_xrVoidFunction*)&pfnGetOpenGLGraphicsRequirementsKHR); + return xrCheck(instance, result, "Failed to get OpenGL graphics requirements function!"); +} + +OpenXrContext::OpenXrContext() { + _isSupported = initPreGraphics(); + if (!_isSupported) { + qCCritical(xr_context_cat, "Pre graphics init failed."); + } +} + +OpenXrContext::~OpenXrContext() { + XrResult res = xrDestroyInstance(_instance); + if (res != XR_SUCCESS) { + qCCritical(xr_context_cat, "Failed to destroy OpenXR instance"); + } + qCDebug(xr_context_cat, "Destroyed instance."); +} + +bool OpenXrContext::initInstance() { + uint32_t count = 0; + XrResult result = xrEnumerateInstanceExtensionProperties(nullptr, 0, &count, nullptr); + + if (!xrCheck(XR_NULL_HANDLE, result, "Failed to enumerate number of extension properties")) + return false; + + std::vector properties; + for (uint32_t i = 0; i < count; i++) { + XrExtensionProperties props = { .type = XR_TYPE_EXTENSION_PROPERTIES }; + properties.push_back(props); + } + + result = xrEnumerateInstanceExtensionProperties(nullptr, count, &count, properties.data()); + if (!xrCheck(XR_NULL_HANDLE, result, "Failed to enumerate extension properties")) + return false; + + bool openglSupported = false; + + qCInfo(xr_context_cat, "Runtime supports %d extensions:", count); + for (uint32_t i = 0; i < count; i++) { + qCInfo(xr_context_cat, "%s v%d", properties[i].extensionName, properties[i].extensionVersion); + if (strcmp(XR_KHR_OPENGL_ENABLE_EXTENSION_NAME, properties[i].extensionName) == 0) { + openglSupported = true; + } + } + + if (!openglSupported) { + qCCritical(xr_context_cat, "Runtime does not support OpenGL!"); + return false; + } + + std::vector enabled = { XR_KHR_OPENGL_ENABLE_EXTENSION_NAME }; + + XrInstanceCreateInfo info = { + .type = XR_TYPE_INSTANCE_CREATE_INFO, + .applicationInfo = { + .applicationName = "overte", + .applicationVersion = 1, + .engineName = "overte", + .engineVersion = 0, + .apiVersion = XR_CURRENT_API_VERSION, + }, + .enabledExtensionCount = (uint32_t)enabled.size(), + .enabledExtensionNames = enabled.data(), + }; + + result = xrCreateInstance(&info, &_instance); + if (!xrCheck(XR_NULL_HANDLE, result, "Failed to create XR instance.")) + return false; + + if (!initFunctionPointers(_instance)) + return false; + + xrStringToPath(_instance, "/user/hand/left", &_handPaths[0]); + xrStringToPath(_instance, "/user/hand/right", &_handPaths[1]); + + return true; +} + +bool OpenXrContext::initSystem() { + XrSystemGetInfo info = { + .type = XR_TYPE_SYSTEM_GET_INFO, + .formFactor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY, + }; + + XrResult result = xrGetSystem(_instance, &info, &_systemId); + if (!xrCheck(_instance, result, "Failed to get system for HMD form factor.")) + return false; + + XrSystemProperties props = { + .type = XR_TYPE_SYSTEM_PROPERTIES, + }; + + result = xrGetSystemProperties(_instance, _systemId, &props); + if (!xrCheck(_instance, result, "Failed to get System properties")) + return false; + + _systemName = QString::fromUtf8(props.systemName); + + qCInfo(xr_context_cat, "System name : %s", props.systemName); + qCInfo(xr_context_cat, "Max layers : %d", props.graphicsProperties.maxLayerCount); + qCInfo(xr_context_cat, "Max swapchain size : %dx%d", props.graphicsProperties.maxSwapchainImageHeight, + props.graphicsProperties.maxSwapchainImageWidth); + qCInfo(xr_context_cat, "Orientation Tracking: %d", props.trackingProperties.orientationTracking); + qCInfo(xr_context_cat, "Position Tracking : %d", props.trackingProperties.positionTracking); + + return true; +} + +bool OpenXrContext::initGraphics() { + XrGraphicsRequirementsOpenGLKHR requirements = { .type = XR_TYPE_GRAPHICS_REQUIREMENTS_OPENGL_KHR }; + XrResult result = pfnGetOpenGLGraphicsRequirementsKHR(_instance, _systemId, &requirements); + return xrCheck(_instance, result, "Failed to get OpenGL graphics requirements!"); +} + +bool OpenXrContext::requestExitSession() { + XrResult result = xrRequestExitSession(_session); + return xrCheck(_instance, result, "Failed to request exit session!"); +} + +bool OpenXrContext::initSession() { + // TODO: Make cross platform + XrGraphicsBindingOpenGLXlibKHR binding = { + .type = XR_TYPE_GRAPHICS_BINDING_OPENGL_XLIB_KHR, + .xDisplay = XOpenDisplay(nullptr), + .glxDrawable = glXGetCurrentDrawable(), + .glxContext = glXGetCurrentContext(), + }; + + XrSessionCreateInfo info = { + .type = XR_TYPE_SESSION_CREATE_INFO, + .next = &binding, + .systemId = _systemId, + }; + + XrResult result = xrCreateSession(_instance, &info, &_session); + return xrCheck(_instance, result, "Failed to create session"); +} + +bool OpenXrContext::initSpaces() { + // TODO: Do xrEnumerateReferenceSpaces before assuming stage space is available. + XrReferenceSpaceCreateInfo stageSpaceInfo = { + .type = XR_TYPE_REFERENCE_SPACE_CREATE_INFO, + .referenceSpaceType = XR_REFERENCE_SPACE_TYPE_STAGE, + .poseInReferenceSpace = XR_INDENTITY_POSE, + }; + + XrResult result = xrCreateReferenceSpace(_session, &stageSpaceInfo, &_stageSpace); + if (!xrCheck(_instance, result, "Failed to create stage space!")) + return false; + + XrReferenceSpaceCreateInfo viewSpaceInfo = { + .type = XR_TYPE_REFERENCE_SPACE_CREATE_INFO, + .referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW, + .poseInReferenceSpace = XR_INDENTITY_POSE, + }; + + result = xrCreateReferenceSpace(_session, &viewSpaceInfo, &_viewSpace); + return xrCheck(_instance, result, "Failed to create view space!"); +} + +#define ENUM_TO_STR(r) \ + case r: \ + return #r + +static std::string xrSessionStateStr(XrSessionState state) { + switch (state) { + ENUM_TO_STR(XR_SESSION_STATE_UNKNOWN); + ENUM_TO_STR(XR_SESSION_STATE_IDLE); + ENUM_TO_STR(XR_SESSION_STATE_READY); + ENUM_TO_STR(XR_SESSION_STATE_SYNCHRONIZED); + ENUM_TO_STR(XR_SESSION_STATE_VISIBLE); + ENUM_TO_STR(XR_SESSION_STATE_FOCUSED); + ENUM_TO_STR(XR_SESSION_STATE_STOPPING); + ENUM_TO_STR(XR_SESSION_STATE_LOSS_PENDING); + ENUM_TO_STR(XR_SESSION_STATE_EXITING); + default: { + std::ostringstream ss; + ss << "UNKNOWN STATE " << state; + return ss.str(); + } + } +} + +// Called before restarting a new session +void OpenXrContext::reset() { + _shouldQuit = false; + _lastSessionState = XR_SESSION_STATE_UNKNOWN; +} + +bool OpenXrContext::updateSessionState(XrSessionState newState) { + qCDebug(xr_context_cat, "Session state changed %s -> %s", xrSessionStateStr(_lastSessionState).c_str(), + xrSessionStateStr(newState).c_str()); + _lastSessionState = newState; + + switch (newState) { + // Don't run frame cycle but keep polling events + case XR_SESSION_STATE_IDLE: + case XR_SESSION_STATE_UNKNOWN: { + _shouldRunFrameCycle = false; + break; + } + + // Run frame cycle and poll events + case XR_SESSION_STATE_FOCUSED: + case XR_SESSION_STATE_SYNCHRONIZED: + case XR_SESSION_STATE_VISIBLE: { + _shouldRunFrameCycle = true; + break; + } + + // Begin the session + case XR_SESSION_STATE_READY: { + if (!_isSessionRunning) { + XrSessionBeginInfo session_begin_info = { + .type = XR_TYPE_SESSION_BEGIN_INFO, + .primaryViewConfigurationType = XR_VIEW_CONFIG_TYPE, + }; + XrResult result = xrBeginSession(_session, &session_begin_info); + if (!xrCheck(_instance, result, "Failed to begin session!")) + return false; + qCDebug(xr_context_cat, "Session started!"); + _isSessionRunning = true; + } + _shouldRunFrameCycle = true; + break; + } + + // End the session, don't render, but keep polling for events + case XR_SESSION_STATE_STOPPING: { + if (_isSessionRunning) { + XrResult result = xrEndSession(_session); + if (!xrCheck(_instance, result, "Failed to end session!")) + return false; + _isSessionRunning = false; + } + _shouldRunFrameCycle = false; + break; + } + + // Destroy session, skip run frame cycle, quit + case XR_SESSION_STATE_LOSS_PENDING: + case XR_SESSION_STATE_EXITING: { + XrResult result = xrDestroySession(_session); + if (!xrCheck(_instance, result, "Failed to destroy session!")) + return false; + _shouldQuit = true; + _shouldRunFrameCycle = false; + qCDebug(xr_context_cat, "Destroyed session"); + break; + } + default: + qCWarning(xr_context_cat, "Unhandled session state: %d", newState); + } + + return true; +} + +bool OpenXrContext::pollEvents() { + XrEventDataBuffer event = { .type = XR_TYPE_EVENT_DATA_BUFFER }; + XrResult result = xrPollEvent(_instance, &event); + while (result == XR_SUCCESS) { + switch (event.type) { + case XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING: { + XrEventDataInstanceLossPending* instanceLossPending = (XrEventDataInstanceLossPending*)&event; + qCCritical(xr_context_cat, "Instance loss pending at %lu! Destroying instance.", instanceLossPending->lossTime); + _shouldQuit = true; + continue; + } + case XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED: { + XrEventDataSessionStateChanged* sessionStateChanged = (XrEventDataSessionStateChanged*)&event; + if (!updateSessionState(sessionStateChanged->state)) { + return false; + } + break; + } + case XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED: { + for (int i = 0; i < HAND_COUNT; i++) { + XrInteractionProfileState state = { .type = XR_TYPE_INTERACTION_PROFILE_STATE }; + XrResult res = xrGetCurrentInteractionProfile(_session, _handPaths[i], &state); + if (!xrCheck(_instance, res, "Failed to get interaction profile")) + continue; + + uint32_t bufferCountOutput; + char profilePath[XR_MAX_PATH_LENGTH]; + res = xrPathToString(_instance, state.interactionProfile, XR_MAX_PATH_LENGTH, &bufferCountOutput, + profilePath); + if (!xrCheck(_instance, res, "Failed to get interaction profile path.")) + continue; + + qCInfo(xr_context_cat, "Controller %d: Interaction profile changed to '%s'", i, profilePath); + } + break; + } + default: + qCWarning(xr_context_cat, "Unhandled event type %d", event.type); + } + + event.type = XR_TYPE_EVENT_DATA_BUFFER; + result = xrPollEvent(_instance, &event); + } + + if (result != XR_EVENT_UNAVAILABLE) { + qCCritical(xr_context_cat, "Failed to poll events!"); + return false; + } + + return true; +} + +bool OpenXrContext::beginFrame() { + XrFrameBeginInfo info = { .type = XR_TYPE_FRAME_BEGIN_INFO }; + XrResult result = xrBeginFrame(_session, &info); + return xrCheck(_instance, result, "failed to begin frame!"); +} + +bool OpenXrContext::initPreGraphics() { + if (!initInstance()) { + return false; + } + + if (!initSystem()) { + return false; + } + + return true; +} + +bool OpenXrContext::initPostGraphics() { + if (!initGraphics()) { + return false; + } + + if (!initSession()) { + return false; + } + + if (!initSpaces()) { + return false; + } + + return true; +} diff --git a/plugins/openxr/src/OpenXrContext.h b/plugins/openxr/src/OpenXrContext.h new file mode 100644 index 0000000000..387bab388e --- /dev/null +++ b/plugins/openxr/src/OpenXrContext.h @@ -0,0 +1,82 @@ +// +// Overte OpenXR Plugin +// +// Copyright 2024 Lubosz Sarnecki +// +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "controllers/Pose.h" + +#include + +#include +#include + +#define HAND_COUNT 2 + +constexpr XrPosef XR_INDENTITY_POSE = { + .orientation = { .x = 0, .y = 0, .z = 0, .w = 1.0 }, + .position = { .x = 0, .y = 0, .z = 0 }, +}; + +constexpr XrViewConfigurationType XR_VIEW_CONFIG_TYPE = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; + +class OpenXrContext { +public: + XrInstance _instance = XR_NULL_HANDLE; + XrSession _session = XR_NULL_HANDLE; + XrSystemId _systemId = XR_NULL_SYSTEM_ID; + + XrSpace _stageSpace = XR_NULL_HANDLE; + XrSpace _viewSpace = XR_NULL_HANDLE; + XrPath _handPaths[HAND_COUNT]; + + controller::Pose _lastHeadPose; + XrTime _lastPredictedDisplayTime; + // TODO: Enable C++17 and use std::optional + bool _lastPredictedDisplayTimeInitialized = false; + + bool _shouldQuit = false; + bool _shouldRunFrameCycle = false; + + bool _isSupported = false; + + QString _systemName; + bool _isSessionRunning = false; + +private: + XrSessionState _lastSessionState = XR_SESSION_STATE_UNKNOWN; + +public: + OpenXrContext(); + ~OpenXrContext(); + + bool initPostGraphics(); + bool beginFrame(); + bool pollEvents(); + bool requestExitSession(); + void reset(); + +private: + bool initPreGraphics(); + bool initInstance(); + bool initSystem(); + bool initGraphics(); + bool initSession(); + bool initSpaces(); + + bool updateSessionState(XrSessionState newState); +}; + +inline static glm::vec3 xrVecToGlm(const XrVector3f& v) { + return glm::vec3(v.x, v.y, v.z); +} + +inline static glm::quat xrQuatToGlm(const XrQuaternionf& q) { + return glm::quat(q.w, q.x, q.y, q.z); +} + +bool xrCheck(XrInstance instance, XrResult result, const char* message); \ No newline at end of file diff --git a/plugins/openxr/src/OpenXrDisplayPlugin.cpp b/plugins/openxr/src/OpenXrDisplayPlugin.cpp new file mode 100644 index 0000000000..e952fe671c --- /dev/null +++ b/plugins/openxr/src/OpenXrDisplayPlugin.cpp @@ -0,0 +1,551 @@ +// +// Overte OpenXR Plugin +// +// Copyright 2024 Lubosz Sarnecki +// +// SPDX-License-Identifier: Apache-2.0 +// + +#include "OpenXrDisplayPlugin.h" +#include + +#include "ViewFrustum.h" + +#include +#include +#include +#include + +Q_DECLARE_LOGGING_CATEGORY(xr_display_cat) +Q_LOGGING_CATEGORY(xr_display_cat, "openxr.display") + +constexpr GLint XR_PREFERRED_COLOR_FORMAT = GL_SRGB8_ALPHA8; + +OpenXrDisplayPlugin::OpenXrDisplayPlugin(std::shared_ptr c) { + _context = c; +} + +bool OpenXrDisplayPlugin::isSupported() const { + return _context->_isSupported; +} + +// Slightly differs from glm::ortho +inline static glm::mat4 fovToProjection(const XrFovf fov, const float near, const float far) { + const float left = tanf(fov.angleLeft); + const float right = tanf(fov.angleRight); + const float down = tanf(fov.angleDown); + const float up = tanf(fov.angleUp); + + const float width = right - left; + const float height = up - down; + + const float m11 = 2 / width; + const float m22 = 2 / height; + const float m33 = -(far + near) / (far - near); + + const float m31 = (right + left) / width; + const float m32 = (up + down) / height; + const float m43 = -(far * (near + near)) / (far - near); + + // clang-format off + const float mat[16] = { + m11, 0 , 0 , 0, + 0 , m22, 0 , 0, + m31, m32, m33, -1, + 0 , 0 , m43, 0, + }; + // clang-format on + + return glm::make_mat4(mat); +} + +glm::mat4 OpenXrDisplayPlugin::getEyeProjection(Eye eye, const glm::mat4& baseProjection) const { + if (!_viewsInitialized) { + return baseProjection; + } + + ViewFrustum frustum; + frustum.setProjection(baseProjection); + return fovToProjection(_views[(eye == Left) ? 0 : 1].fov, frustum.getNearClip(), frustum.getFarClip()); +} + +// TODO: This apparently wasn't right in the OpenVR plugin, but this is what it basically did. +glm::mat4 OpenXrDisplayPlugin::getCullingProjection(const glm::mat4& baseProjection) const { + return getEyeProjection(Left, baseProjection); +} + +// TODO: This should not be explicilty known by the application. +// Let's just render as fast as we can and OpenXR will dictate the pace. +float OpenXrDisplayPlugin::getTargetFrameRate() const { + return std::numeric_limits::max(); +} + +bool OpenXrDisplayPlugin::initViews() { + XrInstance instance = _context->_instance; + XrSystemId systemId = _context->_systemId; + + XrResult result = xrEnumerateViewConfigurationViews(instance, systemId, XR_VIEW_CONFIG_TYPE, 0, &_viewCount, nullptr); + if (!xrCheck(instance, result, "Failed to get view configuration view count!")) { + qCCritical(xr_display_cat, "Failed to get view configuration view count!"); + return false; + } + + assert(_viewCount != 0); + + for (uint32_t i = 0; i < _viewCount; i++) { + XrView view = { .type = XR_TYPE_VIEW }; + _views.push_back(view); + + XrViewConfigurationView viewConfig = { .type = XR_TYPE_VIEW_CONFIGURATION_VIEW }; + _viewConfigs.push_back(viewConfig); + } + + _swapChains.resize(_viewCount); + _swapChainLengths.resize(_viewCount); + _swapChainIndices.resize(_viewCount); + _images.resize(_viewCount); + + result = xrEnumerateViewConfigurationViews(instance, systemId, XR_VIEW_CONFIG_TYPE, _viewCount, &_viewCount, + _viewConfigs.data()); + if (!xrCheck(instance, result, "Failed to enumerate view configuration views!")) { + qCCritical(xr_display_cat, "Failed to enumerate view configuration views!"); + return false; + } + + return true; +} + +#define ENUM_TO_STR(r) \ + case r: \ + return #r + +static std::string glFormatStr(GLenum source) { + switch (source) { + ENUM_TO_STR(GL_RGBA16); + ENUM_TO_STR(GL_RGBA16F); + ENUM_TO_STR(GL_SRGB8_ALPHA8); + default: { + // TODO: Enable C++20 for std::format + std::ostringstream ss; + ss << "0x" << std::hex << source; + return ss.str(); + } + } +} + +static int64_t chooseSwapChainFormat(XrInstance instance, XrSession session, int64_t preferred) { + uint32_t formatCount; + XrResult result = xrEnumerateSwapchainFormats(session, 0, &formatCount, nullptr); + if (!xrCheck(instance, result, "Failed to get number of supported swapchain formats")) + return -1; + + qCInfo(xr_display_cat, "Runtime supports %d swapchain formats", formatCount); + std::vector formats(formatCount); + + result = xrEnumerateSwapchainFormats(session, formatCount, &formatCount, formats.data()); + if (!xrCheck(instance, result, "Failed to enumerate swapchain formats")) + return -1; + + int64_t chosen = formats[0]; + + for (uint32_t i = 0; i < formatCount; i++) { + qCInfo(xr_display_cat, "Supported GL format: %s", glFormatStr(formats[i]).c_str()); + if (formats[i] == preferred) { + chosen = formats[i]; + qCInfo(xr_display_cat, "Using preferred swapchain format %s", glFormatStr(chosen).c_str()); + break; + } + } + if (chosen != preferred) { + qCWarning(xr_display_cat, "Falling back to non preferred swapchain format %s", glFormatStr(chosen).c_str()); + } + + return chosen; +} + +bool OpenXrDisplayPlugin::initSwapChains() { + XrInstance instance = _context->_instance; + XrSession session = _context->_session; + + int64_t format = chooseSwapChainFormat(instance, session, XR_PREFERRED_COLOR_FORMAT); + + for (uint32_t i = 0; i < _viewCount; i++) { + _images[i].clear(); + + XrSwapchainCreateInfo info = { + .type = XR_TYPE_SWAPCHAIN_CREATE_INFO, + .createFlags = 0, + .usageFlags = XR_SWAPCHAIN_USAGE_SAMPLED_BIT | XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT, + .format = format, + .sampleCount = _viewConfigs[i].recommendedSwapchainSampleCount, + .width = _viewConfigs[i].recommendedImageRectWidth, + .height = _viewConfigs[i].recommendedImageRectHeight, + .faceCount = 1, + .arraySize = 1, + .mipCount = 1, + }; + + XrResult result = xrCreateSwapchain(session, &info, &_swapChains[i]); + if (!xrCheck(instance, result, "Failed to create swapchain!")) + return false; + + result = xrEnumerateSwapchainImages(_swapChains[i], 0, &_swapChainLengths[i], nullptr); + if (!xrCheck(instance, result, "Failed to enumerate swapchains")) + return false; + + for (uint32_t j = 0; j < _swapChainLengths[i]; j++) { + XrSwapchainImageOpenGLKHR image = { .type = XR_TYPE_SWAPCHAIN_IMAGE_OPENGL_KHR }; + _images[i].push_back(image); + } + result = xrEnumerateSwapchainImages(_swapChains[i], _swapChainLengths[i], &_swapChainLengths[i], + (XrSwapchainImageBaseHeader*)_images[i].data()); + if (!xrCheck(instance, result, "Failed to enumerate swapchain images")) + return false; + } + + return true; +} + +bool OpenXrDisplayPlugin::initLayers() { + for (uint32_t i = 0; i < _viewCount; i++) { + XrCompositionLayerProjectionView layer = { + .type = XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW, + .subImage = { + .swapchain = _swapChains[i], + .imageRect = { + .offset = { + .x = 0, + .y = 0, + }, + .extent = { + .width = (int32_t)_viewConfigs[i].recommendedImageRectWidth, + .height = (int32_t)_viewConfigs[i].recommendedImageRectHeight, + }, + }, + .imageArrayIndex = 0, + }, + }; + _projectionLayerViews.push_back(layer); + }; + + return true; +} + +void OpenXrDisplayPlugin::init() { + Plugin::init(); + + if (!initViews()) { + qCCritical(xr_display_cat, "View init failed."); + return; + } + + for (const XrViewConfigurationView& view : _viewConfigs) { + assert(view.recommendedImageRectWidth != 0); + qCDebug(xr_display_cat, "Swapchain dimensions: %dx%d", view.recommendedImageRectWidth, view.recommendedImageRectHeight); + // TODO: Don't render side-by-side but use multiview (texture arrays). This probably won't work with GL. + _renderTargetSize.x = view.recommendedImageRectWidth * 2; + _renderTargetSize.y = view.recommendedImageRectHeight; + } + + emit deviceConnected(getName()); +} + +const QString OpenXrDisplayPlugin::getName() const { + return QString("OpenXR: %1").arg(_context->_systemName); +} + +bool OpenXrDisplayPlugin::internalActivate() { + _context->reset(); + return HmdDisplayPlugin::internalActivate(); +} + +void OpenXrDisplayPlugin::internalDeactivate() { + // We can get into a state where activate -> deactivate -> activate is called in a chain. + // We are probably gonna have a bad time then. At least check if the session is already running. + // This happens when the application decides to switch display plugins back and forth. This should + // prbably be fixed there. + if (_context->_isSessionRunning) { + if (!_context->requestExitSession()) { + qCCritical(xr_display_cat, "Failed to request exit session"); + } else { + // Poll events until runtime wants to quit + while (!_context->_shouldQuit) { + _context->pollEvents(); + } + } + } + HmdDisplayPlugin::internalDeactivate(); +} + +void OpenXrDisplayPlugin::customizeContext() { + gl::initModuleGl(); + HmdDisplayPlugin::customizeContext(); + + if (!_context->initPostGraphics()) { + qCCritical(xr_display_cat, "Post graphics init failed."); + return; + } + + if (!initSwapChains()) { + qCCritical(xr_display_cat, "Swap chain init failed."); + return; + } + + if (!initLayers()) { + qCCritical(xr_display_cat, "Layer init failed."); + return; + } + + // Create swap chain images for _compositeFramebuffer + for (size_t i = 0; i < _swapChainLengths[0]; ++i) { + gpu::TexturePointer texture = + gpu::Texture::createRenderBuffer(gpu::Element::COLOR_SRGBA_32, _renderTargetSize.x, _renderTargetSize.y, + gpu::Texture::SINGLE_MIP, gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_POINT)); + _compositeSwapChain.push_back(texture); + } +} + +void OpenXrDisplayPlugin::uncustomizeContext() { + _compositeSwapChain.clear(); + _projectionLayerViews.clear(); + for (uint32_t i = 0; i < _viewCount; i++) { + _images[i].clear(); + } + HmdDisplayPlugin::uncustomizeContext(); +} + +void OpenXrDisplayPlugin::resetSensors() { +} + +bool OpenXrDisplayPlugin::beginFrameRender(uint32_t frameIndex) { + _context->pollEvents(); + + if (_context->_shouldQuit) { + QMetaObject::invokeMethod(qApp, "quit"); + return false; + } + + if (!_context->_shouldRunFrameCycle) { + qCWarning(xr_display_cat, "beginFrameRender: Shoudln't run frame cycle. Skipping renderin frame %d", frameIndex); + return true; + } + + // Wait for present thread + // Actually wait for xrEndFrame to happen. + bool haveFrameToSubmit = true; + { + std::unique_lock lock(_haveFrameMutex); + haveFrameToSubmit = _haveFrameToSubmit; + } + + while (haveFrameToSubmit) { + std::this_thread::sleep_for(std::chrono::microseconds(10)); + { + std::unique_lock lock(_haveFrameMutex); + haveFrameToSubmit = _haveFrameToSubmit; + } + } + + _lastFrameState = { .type = XR_TYPE_FRAME_STATE }; + XrResult result = xrWaitFrame(_context->_session, nullptr, &_lastFrameState); + + if (!xrCheck(_context->_instance, result, "xrWaitFrame failed")) + return false; + + if (!_context->beginFrame()) + return false; + + _context->_lastPredictedDisplayTime = _lastFrameState.predictedDisplayTime; + _context->_lastPredictedDisplayTimeInitialized = true; + + std::vector eye_views(_viewCount); + for (uint32_t i = 0; i < _viewCount; i++) { + eye_views[i].type = XR_TYPE_VIEW; + } + + // TODO: Probably shouldn't call xrLocateViews twice. Use only view space views? + XrViewLocateInfo eyeViewLocateInfo = { + .type = XR_TYPE_VIEW_LOCATE_INFO, + .viewConfigurationType = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, + .displayTime = _lastFrameState.predictedDisplayTime, + .space = _context->_viewSpace, + }; + + XrViewState eyeViewState = { .type = XR_TYPE_VIEW_STATE }; + + result = xrLocateViews(_context->_session, &eyeViewLocateInfo, &eyeViewState, _viewCount, &_viewCount, eye_views.data()); + if (!xrCheck(_context->_instance, result, "Could not locate views")) + return false; + + for (uint32_t i = 0; i < 2; i++) { + vec3 eyePosition = xrVecToGlm(eye_views[i].pose.position); + quat eyeOrientation = xrQuatToGlm(eye_views[i].pose.orientation); + _eyeOffsets[i] = controller::Pose(eyePosition, eyeOrientation).getMatrix(); + } + + _lastViewState = { .type = XR_TYPE_VIEW_STATE }; + + XrViewLocateInfo viewLocateInfo = { + .type = XR_TYPE_VIEW_LOCATE_INFO, + .viewConfigurationType = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, + .displayTime = _lastFrameState.predictedDisplayTime, + .space = _context->_stageSpace, + }; + + result = xrLocateViews(_context->_session, &viewLocateInfo, &_lastViewState, _viewCount, &_viewCount, _views.data()); + if (!xrCheck(_context->_instance, result, "Could not locate views")) + return false; + + for (uint32_t i = 0; i < _viewCount; i++) { + _projectionLayerViews[i].pose = _views[i].pose; + _projectionLayerViews[i].fov = _views[i].fov; + } + + _viewsInitialized = true; + + XrSpaceLocation headLocation = { + .type = XR_TYPE_SPACE_LOCATION, + .pose = XR_INDENTITY_POSE, + }; + xrLocateSpace(_context->_viewSpace, _context->_stageSpace, _lastFrameState.predictedDisplayTime, &headLocation); + + glm::vec3 headPosition = xrVecToGlm(headLocation.pose.position); + glm::quat headOrientation = xrQuatToGlm(headLocation.pose.orientation); + _context->_lastHeadPose = controller::Pose(headPosition, headOrientation); + + _currentRenderFrameInfo = FrameInfo(); + _currentRenderFrameInfo.renderPose = _context->_lastHeadPose.getMatrix(); + _currentRenderFrameInfo.presentPose = _currentRenderFrameInfo.renderPose; + _frameInfos[frameIndex] = _currentRenderFrameInfo; + + return HmdDisplayPlugin::beginFrameRender(frameIndex); +} + +void OpenXrDisplayPlugin::submitFrame(const gpu::FramePointer& newFrame) { + OpenGLDisplayPlugin::submitFrame(newFrame); + { + std::unique_lock lock(_haveFrameMutex); + _haveFrameToSubmit = true; + } +} + +void OpenXrDisplayPlugin::compositeLayers() { + if (!_context->_shouldRunFrameCycle) { + return; + } + + if (_lastFrameState.shouldRender) { + _compositeFramebuffer->setRenderBuffer(0, _compositeSwapChain[_swapChainIndices[0]]); + HmdDisplayPlugin::compositeLayers(); + } +} + +void OpenXrDisplayPlugin::hmdPresent() { + if (!_context->_shouldRunFrameCycle) { + qCWarning(xr_display_cat, "hmdPresent: Shoudln't run frame cycle. Skipping renderin frame %d", + _currentFrame->frameIndex); + return; + } + + if (_lastFrameState.shouldRender) { + // TODO: Use multiview swapchain + for (uint32_t i = 0; i < 2; i++) { + XrSwapchainImageAcquireInfo acquireInfo = { .type = XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO }; + + XrResult result = xrAcquireSwapchainImage(_swapChains[i], &acquireInfo, &_swapChainIndices[i]); + if (!xrCheck(_context->_instance, result, "failed to acquire swapchain image!")) + return; + + XrSwapchainImageWaitInfo waitInfo = { .type = XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO, .timeout = 1000 }; + result = xrWaitSwapchainImage(_swapChains[i], &waitInfo); + if (!xrCheck(_context->_instance, result, "failed to wait for swapchain image!")) + return; + } + + GLuint glTexId = getGLBackend()->getTextureID(_compositeFramebuffer->getRenderBuffer(0)); + + glCopyImageSubData(glTexId, GL_TEXTURE_2D, 0, 0, 0, 0, _images[0][_swapChainIndices[0]].image, GL_TEXTURE_2D, 0, 0, 0, + 0, _renderTargetSize.x / 2, _renderTargetSize.y, 1); + + glCopyImageSubData(glTexId, GL_TEXTURE_2D, 0, _renderTargetSize.x / 2, 0, 0, _images[1][_swapChainIndices[1]].image, + GL_TEXTURE_2D, 0, 0, 0, 0, _renderTargetSize.x / 2, _renderTargetSize.y, 1); + + for (uint32_t i = 0; i < 2; i++) { + XrSwapchainImageReleaseInfo releaseInfo = { .type = XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO }; + XrResult result = xrReleaseSwapchainImage(_swapChains[i], &releaseInfo); + if (!xrCheck(_context->_instance, result, "failed to release swapchain image!")) { + assert(false); + return; + } + } + } + + endFrame(); + + _presentRate.increment(); + + { + std::unique_lock lock(_haveFrameMutex); + _haveFrameToSubmit = false; + } +} + +bool OpenXrDisplayPlugin::endFrame() { + XrCompositionLayerProjection projectionLayer = { + .type = XR_TYPE_COMPOSITION_LAYER_PROJECTION, + .layerFlags = 0, + .space = _context->_stageSpace, + .viewCount = _viewCount, + .views = _projectionLayerViews.data(), + }; + + std::vector layers = { + (const XrCompositionLayerBaseHeader*)&projectionLayer, + }; + + XrFrameEndInfo info = { + .type = XR_TYPE_FRAME_END_INFO, + .displayTime = _lastFrameState.predictedDisplayTime, + .environmentBlendMode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE, + .layerCount = (uint32_t)layers.size(), + .layers = layers.data(), + }; + + if ((_lastViewState.viewStateFlags & XR_VIEW_STATE_ORIENTATION_VALID_BIT) == 0) { + qCWarning(xr_display_cat, "Not submitting layers because orientation is invalid."); + info.layerCount = 0; + } + + if (!_lastFrameState.shouldRender) { + info.layerCount = 0; + } + + XrResult result = xrEndFrame(_context->_session, &info); + if (!xrCheck(_context->_instance, result, "failed to end frame!")) { + return false; + } + + return true; +} + +void OpenXrDisplayPlugin::postPreview() { +} + +bool OpenXrDisplayPlugin::isHmdMounted() const { + return true; +} + +void OpenXrDisplayPlugin::updatePresentPose() { +} + +int OpenXrDisplayPlugin::getRequiredThreadCount() const { + return HmdDisplayPlugin::getRequiredThreadCount(); +} + +QRectF OpenXrDisplayPlugin::getPlayAreaRect() { + return QRectF(0, 0, 10, 10); +} + +DisplayPlugin::StencilMaskMeshOperator OpenXrDisplayPlugin::getStencilMaskMeshOperator() { + return nullptr; +} diff --git a/plugins/openxr/src/OpenXrDisplayPlugin.h b/plugins/openxr/src/OpenXrDisplayPlugin.h new file mode 100644 index 0000000000..e36dba44f5 --- /dev/null +++ b/plugins/openxr/src/OpenXrDisplayPlugin.h @@ -0,0 +1,97 @@ +// +// Overte OpenXR Plugin +// +// Copyright 2024 Lubosz Sarnecki +// +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "OpenXrContext.h" + +#include "gpu/gl/GLBackend.h" + +#include + +#define XR_USE_PLATFORM_XLIB +#define XR_USE_GRAPHICS_API_OPENGL +#include +#include + +class OpenXrDisplayPlugin : public HmdDisplayPlugin { +public: + OpenXrDisplayPlugin(std::shared_ptr c); + bool isSupported() const override; + const QString getName() const override; + bool getSupportsAutoSwitch() override final { return true; } + + glm::mat4 getEyeProjection(Eye eye, const glm::mat4& baseProjection) const override; + glm::mat4 getCullingProjection(const glm::mat4& baseProjection) const override; + + void init() override; + + float getTargetFrameRate() const override; + bool hasAsyncReprojection() const override { return true; } + + void customizeContext() override; + void uncustomizeContext() override; + + void resetSensors() override; + bool beginFrameRender(uint32_t frameIndex) override; + void submitFrame(const gpu::FramePointer& newFrame) override; + void cycleDebugOutput() override { _lockCurrentTexture = !_lockCurrentTexture; } + + int getRequiredThreadCount() const override; + + QRectF getPlayAreaRect() override; + + virtual StencilMaskMode getStencilMaskMode() const override { return StencilMaskMode::MESH; } + virtual StencilMaskMeshOperator getStencilMaskMeshOperator() override; + + glm::mat4 getSensorResetMatrix() const { return glm::mat4(1.0f); } + +protected: + bool internalActivate() override; + void internalDeactivate() override; + void updatePresentPose() override; + + void compositeLayers() override; + void hmdPresent() override; + bool isHmdMounted() const override; + void postPreview() override; + +private: + std::vector _compositeSwapChain; + + XrViewState _lastViewState; + + std::shared_ptr _context; + + uint32_t _viewCount = 0; + std::vector _projectionLayerViews; + + std::vector _views; + // TODO: Enable C++17 and use std::optional + bool _viewsInitialized = false; + + std::vector _viewConfigs; + + std::vector _swapChains; + std::vector _swapChainLengths; + std::vector _swapChainIndices; + std::vector> _images; + + XrFrameState _lastFrameState; + + bool initViews(); + bool initSwapChains(); + bool initLayers(); + bool endFrame(); + + bool _haveFrameToSubmit = false; + std::mutex _haveFrameMutex; +}; diff --git a/plugins/openxr/src/OpenXrInputPlugin.cpp b/plugins/openxr/src/OpenXrInputPlugin.cpp new file mode 100644 index 0000000000..f5eaabff0b --- /dev/null +++ b/plugins/openxr/src/OpenXrInputPlugin.cpp @@ -0,0 +1,523 @@ +// +// Overte OpenXR Plugin +// +// Copyright 2024 Lubosz Sarnecki +// +// SPDX-License-Identifier: Apache-2.0 +// + +#include "OpenXrInputPlugin.h" + +#include "AvatarConstants.h" +#include "PathUtils.h" + +#include "controllers/UserInputMapper.h" + +Q_DECLARE_LOGGING_CATEGORY(xr_input_cat) +Q_LOGGING_CATEGORY(xr_input_cat, "openxr.input") + +OpenXrInputPlugin::OpenXrInputPlugin(std::shared_ptr c) { + _context = c; + _inputDevice = std::make_shared(_context); +} + +// TODO: Make a config UI +static const QString XR_CONFIGURATION_LAYOUT = QString(""); + +void OpenXrInputPlugin::calibrate() { +} + +bool OpenXrInputPlugin::uncalibrate() { + return true; +} + +bool OpenXrInputPlugin::isSupported() const { + return _context->_isSupported; +} + +void OpenXrInputPlugin::setConfigurationSettings(const QJsonObject configurationSettings) { +} + +QJsonObject OpenXrInputPlugin::configurationSettings() { + return QJsonObject(); +} + +QString OpenXrInputPlugin::configurationLayout() { + return XR_CONFIGURATION_LAYOUT; +} + +bool OpenXrInputPlugin::activate() { + InputPlugin::activate(); + + loadSettings(); + + // register with UserInputMapper + auto userInputMapper = DependencyManager::get(); + userInputMapper->registerDevice(_inputDevice); + _registeredWithInputMapper = true; + + return true; +} + +void OpenXrInputPlugin::deactivate() { + InputPlugin::deactivate(); + + _inputDevice->_poseStateMap.clear(); + + // unregister with UserInputMapper + auto userInputMapper = DependencyManager::get(); + userInputMapper->removeDevice(_inputDevice->_deviceID); + _registeredWithInputMapper = false; + + saveSettings(); +} + +void OpenXrInputPlugin::pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) { + if (_context->_shouldQuit) { + deactivate(); + return; + } + + auto userInputMapper = DependencyManager::get(); + userInputMapper->withLock([&, this]() { _inputDevice->update(deltaTime, inputCalibrationData); }); + + if (_inputDevice->_trackedControllers == 0 && _registeredWithInputMapper) { + userInputMapper->removeDevice(_inputDevice->_deviceID); + _registeredWithInputMapper = false; + _inputDevice->_poseStateMap.clear(); + } + + if (!_registeredWithInputMapper && _inputDevice->_trackedControllers > 0) { + userInputMapper->registerDevice(_inputDevice); + _registeredWithInputMapper = true; + } +} + +void OpenXrInputPlugin::loadSettings() { +} + +void OpenXrInputPlugin::saveSettings() const { +} + +OpenXrInputPlugin::InputDevice::InputDevice(std::shared_ptr c) : controller::InputDevice("Index") { + _context = c; +} + +void OpenXrInputPlugin::InputDevice::focusOutEvent() { + _axisStateMap.clear(); + _buttonPressedMap.clear(); +}; + +bool OpenXrInputPlugin::InputDevice::triggerHapticPulse(float strength, float duration, uint16_t index) { + if (index > 2) { + return false; + } + + std::unique_lock locker(_lock); + + // TODO: convert duration and strength to openxr values. + if (!_actions.at("/output/haptic")->applyHaptic(0, XR_MIN_HAPTIC_DURATION, XR_FREQUENCY_UNSPECIFIED, 0.5f)) { + qCCritical(xr_input_cat, "Failed to apply haptic feedback!"); + } + + return true; +} + +bool OpenXrInputPlugin::Action::init(XrActionSet actionSet) { + XrInstance instance = _context->_instance; + XrActionCreateInfo info = { + .type = XR_TYPE_ACTION_CREATE_INFO, + .actionType = _type, + .countSubactionPaths = HAND_COUNT, + .subactionPaths = _context->_handPaths, + }; + + QString name = QString::fromStdString(_path); + name.replace("/input/", ""); + name.replace("/", "-"); + strcpy(info.actionName, name.toUtf8().data()); + name.replace("-", " "); + strcpy(info.localizedActionName, name.toUtf8().data()); + + XrResult result = xrCreateAction(actionSet, &info, &_action); + if (!xrCheck(instance, result, "Failed to create action")) + return false; + + // Pose actions need spaces + if (_type == XR_ACTION_TYPE_POSE_INPUT) { + if (!createPoseSpaces()) { + return false; + } + } + + return true; +} + +const std::vector HAND_PATHS = { "left", "right" }; + +std::vector OpenXrInputPlugin::Action::getBindings() { + assert(_action != XR_NULL_HANDLE); + + std::vector bindings; + for (uint32_t i = 0; i < HAND_COUNT; i++) { + XrPath path; + std::string pathString = "/user/hand/" + HAND_PATHS[i] + _path; + xrStringToPath(_context->_instance, pathString.c_str(), &path); + XrActionSuggestedBinding binding = { .action = _action, .binding = path }; + bindings.push_back(binding); + } + return bindings; +} + +XrActionStateFloat OpenXrInputPlugin::Action::getFloat(uint32_t handId) { + XrActionStateFloat state = { + .type = XR_TYPE_ACTION_STATE_FLOAT, + }; + + XrActionStateGetInfo info = { + .type = XR_TYPE_ACTION_STATE_GET_INFO, + .action = _action, + .subactionPath = _context->_handPaths[handId], + }; + + XrResult result = xrGetActionStateFloat(_context->_session, &info, &state); + xrCheck(_context->_instance, result, "Failed to get float state!"); + + return state; +} + +XrActionStateBoolean OpenXrInputPlugin::Action::getBool(uint32_t handId) { + XrActionStateBoolean state = { + .type = XR_TYPE_ACTION_STATE_BOOLEAN, + }; + + XrActionStateGetInfo info = { + .type = XR_TYPE_ACTION_STATE_GET_INFO, + .action = _action, + .subactionPath = _context->_handPaths[handId], + }; + + XrResult result = xrGetActionStateBoolean(_context->_session, &info, &state); + xrCheck(_context->_instance, result, "Failed to get float state!"); + + return state; +} + +XrSpaceLocation OpenXrInputPlugin::Action::getPose(uint32_t handId) { + XrActionStatePose state = { + .type = XR_TYPE_ACTION_STATE_POSE, + }; + XrActionStateGetInfo info = { + .type = XR_TYPE_ACTION_STATE_GET_INFO, + .action = _action, + .subactionPath = _context->_handPaths[handId], + }; + + XrResult result = xrGetActionStatePose(_context->_session, &info, &state); + xrCheck(_context->_instance, result, "failed to get pose value!"); + + XrSpaceLocation location = { + .type = XR_TYPE_SPACE_LOCATION, + }; + + if (_context->_lastPredictedDisplayTimeInitialized) { + result = xrLocateSpace(_poseSpaces[handId], _context->_stageSpace, _context->_lastPredictedDisplayTime, &location); + xrCheck(_context->_instance, result, "Failed to locate hand space!"); + } + + return location; +} + +bool OpenXrInputPlugin::Action::applyHaptic(uint32_t handId, XrDuration duration, float frequency, float amplitude) { + XrHapticVibration vibration = { + .type = XR_TYPE_HAPTIC_VIBRATION, + .duration = duration, + .frequency = frequency, + .amplitude = amplitude, + }; + + XrHapticActionInfo haptic_action_info = { + .type = XR_TYPE_HAPTIC_ACTION_INFO, + .action = _action, + .subactionPath = _context->_handPaths[handId], + }; + XrResult result = xrApplyHapticFeedback(_context->_session, &haptic_action_info, (const XrHapticBaseHeader*)&vibration); + + return xrCheck(_context->_instance, result, "Failed to apply haptic feedback!"); +} + +bool OpenXrInputPlugin::Action::createPoseSpaces() { + assert(_action != XR_NULL_HANDLE); + + for (int hand = 0; hand < HAND_COUNT; hand++) { + XrActionSpaceCreateInfo info = { + .type = XR_TYPE_ACTION_SPACE_CREATE_INFO, + .action = _action, + .subactionPath = _context->_handPaths[hand], + .poseInActionSpace = XR_INDENTITY_POSE, + }; + + XrResult result = xrCreateActionSpace(_context->_session, &info, &_poseSpaces[hand]); + if (!xrCheck(_context->_instance, result, "Failed to create hand pose space")) + return false; + } + + return true; +} + +bool OpenXrInputPlugin::InputDevice::initBindings(const std::string& profileName, + const std::vector& actionsToBind) { + XrPath profilePath; + XrResult result = xrStringToPath(_context->_instance, profileName.c_str(), &profilePath); + if (!xrCheck(_context->_instance, result, "Failed to get interaction profile")) + return false; + + std::vector bindings; + for (const std::string& path : actionsToBind) { + std::vector actionBindings = _actions.at(path)->getBindings(); + bindings.insert(std::end(bindings), std::begin(actionBindings), std::end(actionBindings)); + } + + const XrInteractionProfileSuggestedBinding suggestedBinding = { + .type = XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING, + .interactionProfile = profilePath, + .countSuggestedBindings = (uint32_t)bindings.size(), + .suggestedBindings = bindings.data(), + }; + + result = xrSuggestInteractionProfileBindings(_context->_instance, &suggestedBinding); + + return xrCheck(_context->_instance, result, "Failed to suggest bindings"); +} + +controller::Input::NamedVector OpenXrInputPlugin::InputDevice::getAvailableInputs() const { + using namespace controller; + + // clang-format off + QVector availableInputs{ + // Poses + makePair(LEFT_HAND, "LeftHand"), + makePair(RIGHT_HAND, "RightHand"), + makePair(HEAD, "Head"), + // Sticks + makePair(LX, "LX"), + makePair(LY, "LY"), + makePair(RX, "RX"), + makePair(RY, "RY"), + // Face buttons + makePair(A, "A"), + makePair(B, "B"), + makePair(X, "X"), + makePair(Y, "Y"), + // Triggers + makePair(RT, "RT"), + makePair(LT, "LT"), + makePair(RT_CLICK, "RTClick"), + makePair(LT_CLICK, "LTClick"), + // Menu buttons + // TODO: Add this to button channel + // Input::NamedPair(Input(_deviceID, LEFT_APP_MENU, ChannelType::BUTTON), "LeftApplicationMenu"), + // Input::NamedPair(Input(_deviceID, RIGHT_APP_MENU, ChannelType::BUTTON), "RightApplicationMenu"), + }; + // clang-format on + + return availableInputs; +} + +QString OpenXrInputPlugin::InputDevice::getDefaultMappingConfig() const { + return PathUtils::resourcesPath() + "/controllers/openxr_index.json"; +} + +bool OpenXrInputPlugin::InputDevice::initActions() { + if (_actionsInitialized) + return true; + + assert(_context->_session != XR_NULL_HANDLE); + + XrInstance instance = _context->_instance; + + XrActionSetCreateInfo actionSetInfo = { + .type = XR_TYPE_ACTION_SET_CREATE_INFO, + .actionSetName = "action_set", + .localizedActionSetName = "Action Set", + .priority = 0, + }; + XrResult result = xrCreateActionSet(instance, &actionSetInfo, &_actionSet); + if (!xrCheck(instance, result, "Failed to create action set.")) + return false; + + // clang-format off + std::map actionsToInit = { + { "/input/thumbstick/x", XR_ACTION_TYPE_FLOAT_INPUT }, + { "/input/thumbstick/y", XR_ACTION_TYPE_FLOAT_INPUT }, + { "/input/a/click", XR_ACTION_TYPE_BOOLEAN_INPUT }, + { "/input/b/click", XR_ACTION_TYPE_BOOLEAN_INPUT }, + { "/input/trigger/value", XR_ACTION_TYPE_FLOAT_INPUT }, + { "/input/trigger/click", XR_ACTION_TYPE_BOOLEAN_INPUT }, + { "/output/haptic", XR_ACTION_TYPE_VIBRATION_OUTPUT }, + { "/input/grip/pose", XR_ACTION_TYPE_POSE_INPUT }, + { "/input/select/click", XR_ACTION_TYPE_BOOLEAN_INPUT }, + { "/input/system/click", XR_ACTION_TYPE_BOOLEAN_INPUT }, + }; + // clang-format on + + for (const auto& pathAndType : actionsToInit) { + std::shared_ptr action = std::make_shared(_context, pathAndType.second, pathAndType.first); + if (!action->init(_actionSet)) { + qCCritical(xr_input_cat, "Creating action %s failed!", pathAndType.first.c_str()); + } else { + _actions.emplace(pathAndType.first, action); + } + } + + // Khronos Simple Controller + std::vector simpleBindings = { + "/input/grip/pose", + "/input/select/click", + "/output/haptic", + }; + + if (!initBindings("/interaction_profiles/khr/simple_controller", simpleBindings)) { + qCCritical(xr_input_cat, "Failed to init bindings."); + } + + // Valve Index Controller + // clang-format off + std::vector indexBindings = { + "/input/grip/pose", + "/input/thumbstick/x", + "/input/thumbstick/y", + "/input/a/click", + "/input/b/click", + "/input/trigger/value", + "/output/haptic", + "/input/system/click", + }; + // clang-format on + + if (!initBindings("/interaction_profiles/valve/index_controller", indexBindings)) { + qCCritical(xr_input_cat, "Failed to init bindings."); + } + + XrSessionActionSetsAttachInfo attachInfo = { + .type = XR_TYPE_SESSION_ACTION_SETS_ATTACH_INFO, + .countActionSets = 1, + .actionSets = &_actionSet, + }; + result = xrAttachSessionActionSets(_context->_session, &attachInfo); + if (!xrCheck(_context->_instance, result, "Failed to attach action set")) + return false; + + _actionsInitialized = true; + + return true; +} + +void OpenXrInputPlugin::InputDevice::update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) { + _poseStateMap.clear(); + _buttonPressedMap.clear(); + _trackedControllers = 2; + + if (_context->_session == XR_NULL_HANDLE) { + return; + } + + if (!initActions()) { + qCCritical(xr_input_cat, "Could not initialize actions!"); + return; + } + + const XrActiveActionSet active_actionset = { + .actionSet = _actionSet, + }; + + XrActionsSyncInfo syncInfo = { + .type = XR_TYPE_ACTIONS_SYNC_INFO, + .countActiveActionSets = 1, + .activeActionSets = &active_actionset, + }; + + XrInstance instance = _context->_instance; + XrSession session = _context->_session; + + XrResult result = xrSyncActions(session, &syncInfo); + xrCheck(instance, result, "failed to sync actions!"); + + glm::mat4 sensorToAvatar = glm::inverse(inputCalibrationData.avatarMat) * inputCalibrationData.sensorToWorldMat; + + static const glm::quat yFlip = glm::angleAxis(PI, Vectors::UNIT_Y); + static const glm::quat quarterX = glm::angleAxis(PI_OVER_TWO, Vectors::UNIT_X); + static const glm::quat touchToHand = yFlip * quarterX; + + static const glm::quat leftQuarterZ = glm::angleAxis(-PI_OVER_TWO, Vectors::UNIT_Z); + static const glm::quat rightQuarterZ = glm::angleAxis(PI_OVER_TWO, Vectors::UNIT_Z); + static const glm::quat eighthX = glm::angleAxis(PI / 4.0f, Vectors::UNIT_X); + + static const glm::quat leftRotationOffset = glm::inverse(leftQuarterZ * eighthX) * touchToHand; + static const glm::quat rightRotationOffset = glm::inverse(rightQuarterZ * eighthX) * touchToHand; + + for (int i = 0; i < HAND_COUNT; i++) { + XrSpaceLocation handLocation = _actions.at("/input/grip/pose")->getPose(i); + bool locationValid = (handLocation.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT) != 0; + if (locationValid) { + vec3 translation = xrVecToGlm(handLocation.pose.position); + quat rotation = xrQuatToGlm(handLocation.pose.orientation); + auto pose = controller::Pose(translation, rotation); + glm::mat4 handOffset = i == 0 ? glm::toMat4(leftRotationOffset) : glm::toMat4(rightRotationOffset); + _poseStateMap[i == 0 ? controller::LEFT_HAND : controller::RIGHT_HAND] = + pose.postTransform(handOffset).transform(sensorToAvatar); + } + } + + glm::mat4 defaultHeadOffset = createMatFromQuatAndPos(-DEFAULT_AVATAR_HEAD_ROT, -DEFAULT_AVATAR_HEAD_TO_MIDDLE_EYE_OFFSET); + _poseStateMap[controller::HEAD] = _context->_lastHeadPose.postTransform(defaultHeadOffset).transform(sensorToAvatar); + + std::map axesToUpdate[2] = { + { + { controller::LX, "/input/thumbstick/x" }, + { controller::LY, "/input/thumbstick/y" }, + { controller::LT, "/input/trigger/value" }, + }, + { + { controller::RX, "/input/thumbstick/x" }, + { controller::RY, "/input/thumbstick/y" }, + { controller::RT, "/input/trigger/value" }, + }, + }; + + for (uint32_t i = 0; i < HAND_COUNT; i++) { + for (const auto& channelAndPath : axesToUpdate[i]) { + _axisStateMap[channelAndPath.first].value = _actions.at(channelAndPath.second)->getFloat(i).currentState; + + // if (_axisStateMap[channelAndPath.first].value != 0) { + // qCDebug(xr_input_cat, "🐸 Controller %d: %s (%d): %f", i, channelAndPath.second.c_str(), channelAndPath.first, + // (double)_axisStateMap[channelAndPath.first].value); + // } + } + } + + // TODO: Figure out why LEFT_APP_MENU is misssing in StandardButtonChannel + std::map buttonsToUpdate[2] = { + { + { controller::X, "/input/a/click" }, + { controller::Y, "/input/b/click" }, + { controller::LT_CLICK, "/input/trigger/click" }, + //{ LEFT_APP_MENU, "/input/system/click" }, + }, + { + { controller::A, "/input/a/click" }, + { controller::B, "/input/b/click" }, + { controller::RT_CLICK, "/input/trigger/click" }, + //{ RIGHT_APP_MENU, "/input/system/click" }, + }, + }; + + for (uint32_t i = 0; i < HAND_COUNT; i++) { + for (const auto& channelAndPath : buttonsToUpdate[i]) { + if (_actions.at(channelAndPath.second)->getBool(i).currentState == XR_TRUE) { + _buttonPressedMap.insert(channelAndPath.first); + // qCDebug(xr_input_cat, "🐸 Controller %d: %s (%d)", i, channelAndPath.second.c_str(), channelAndPath.first); + } + } + } +} \ No newline at end of file diff --git a/plugins/openxr/src/OpenXrInputPlugin.h b/plugins/openxr/src/OpenXrInputPlugin.h new file mode 100644 index 0000000000..f1a578867d --- /dev/null +++ b/plugins/openxr/src/OpenXrInputPlugin.h @@ -0,0 +1,104 @@ +// +// Overte OpenXR Plugin +// +// Copyright 2024 Lubosz Sarnecki +// +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "plugins/InputPlugin.h" +#include "controllers/InputDevice.h" +#include "OpenXrContext.h" + +#define HAND_COUNT 2 + +class OpenXrInputPlugin : public InputPlugin { + Q_OBJECT +public: + OpenXrInputPlugin(std::shared_ptr c); + bool isSupported() const override; + const QString getName() const override { return "OpenXR"; } + + bool isHandController() const override { return true; } + bool configurable() override { return true; } + + QString configurationLayout() override; + void setConfigurationSettings(const QJsonObject configurationSettings) override; + QJsonObject configurationSettings() override; + void calibrate() override; + bool uncalibrate() override; + bool isHeadController() const override { return true; } + + bool activate() override; + void deactivate() override; + + QString getDeviceName() override { return _context.get()->_systemName; } + + void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); } + + void pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override; + + virtual void saveSettings() const override; + virtual void loadSettings() override; + +private: + class Action { + public: + Action(std::shared_ptr c, XrActionType type, const std::string& path) { + _context = c; + _path = path; + _type = type; + } + + bool init(XrActionSet actionSet); + std::vector getBindings(); + XrActionStateFloat getFloat(uint32_t handId); + XrActionStateBoolean getBool(uint32_t handId); + XrSpaceLocation getPose(uint32_t handId); + bool applyHaptic(uint32_t handId, XrDuration duration, float frequency, float amplitude); + + private: + bool createPoseSpaces(); + XrAction _action = XR_NULL_HANDLE; + std::shared_ptr _context; + std::string _path; + XrActionType _type; + XrSpace _poseSpaces[HAND_COUNT] = { XR_NULL_HANDLE, XR_NULL_HANDLE }; + }; + + class InputDevice : public controller::InputDevice { + public: + InputDevice(std::shared_ptr c); + + private: + controller::Input::NamedVector getAvailableInputs() const override; + QString getDefaultMappingConfig() const override; + void update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override; + void focusOutEvent() override; + bool triggerHapticPulse(float strength, float duration, uint16_t index) override; + + mutable std::recursive_mutex _lock; + template + void withLock(F&& f) { + std::unique_lock locker(_lock); + f(); + } + + friend class OpenXrInputPlugin; + + uint32_t _trackedControllers = 0; + XrActionSet _actionSet; + std::map> _actions; + std::shared_ptr _context; + bool _actionsInitialized = false; + + bool initActions(); + bool initBindings(const std::string& profileName, const std::vector& actionsToBind); + }; + + bool _registeredWithInputMapper = false; + std::shared_ptr _context; + std::shared_ptr _inputDevice; +}; diff --git a/plugins/openxr/src/OpenXrProvider.cpp b/plugins/openxr/src/OpenXrProvider.cpp new file mode 100644 index 0000000000..c2a87dff67 --- /dev/null +++ b/plugins/openxr/src/OpenXrProvider.cpp @@ -0,0 +1,59 @@ +// +// Overte OpenXR Plugin +// +// Copyright 2024 Lubosz Sarnecki +// +// SPDX-License-Identifier: Apache-2.0 +// + +#include "plugins/RuntimePlugin.h" +#include "OpenXrDisplayPlugin.h" +#include "OpenXrInputPlugin.h" + +class OpenXrProvider : public QObject, public DisplayProvider, InputProvider { + Q_OBJECT + Q_PLUGIN_METADATA(IID DisplayProvider_iid FILE "plugin.json") + Q_INTERFACES(DisplayProvider) + Q_PLUGIN_METADATA(IID InputProvider_iid FILE "plugin.json") + Q_INTERFACES(InputProvider) + +public: + OpenXrProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~OpenXrProvider() {} + std::shared_ptr context = std::make_shared(); + + virtual DisplayPluginList getDisplayPlugins() override { + static std::once_flag once; + std::call_once(once, [&] { + DisplayPluginPointer plugin(std::make_shared(context)); + if (plugin->isSupported()) { + _displayPlugins.push_back(plugin); + } + }); + + return _displayPlugins; + } + + virtual InputPluginList getInputPlugins() override { + static std::once_flag once; + + std::call_once(once, [&] { + InputPluginPointer plugin(std::make_shared(context)); + if (plugin->isSupported()) { + _inputPlugins.push_back(plugin); + } + }); + + return _inputPlugins; + } + + virtual void destroyInputPlugins() override { _inputPlugins.clear(); } + + virtual void destroyDisplayPlugins() override { _displayPlugins.clear(); } + +private: + DisplayPluginList _displayPlugins; + InputPluginList _inputPlugins; +}; + +#include "OpenXrProvider.moc" diff --git a/plugins/openxr/src/plugin.json b/plugins/openxr/src/plugin.json new file mode 100644 index 0000000000..5a3df6e736 --- /dev/null +++ b/plugins/openxr/src/plugin.json @@ -0,0 +1,4 @@ +{ + "name":"OpenXR", + "version":1 +}