Merge pull request #8026 from jherico/steamvr_keyboard

SteamVR keyboard functionality

libraries/gl/src/gl/OffscreenQmlSurface.cpp

@@ -396,6 +396,8 @@ void OffscreenQmlSurface::create(QOpenGLContext* shareContext) {
 
     _renderer->_renderControl->_renderWindow = _proxyWindow;
 
+    connect(_renderer->_quickWindow, &QQuickWindow::focusObjectChanged, this, &OffscreenQmlSurface::onFocusObjectChanged);
+
     // Create a QML engine.
     _qmlEngine = new QQmlEngine;
     if (!_qmlEngine->incubationController()) {
@@ -742,3 +744,21 @@ QVariant OffscreenQmlSurface::returnFromUiThread(std::function<QVariant()> funct
 
     return function();
 }
+
+void OffscreenQmlSurface::onFocusObjectChanged(QObject* object) {
+    if (!object) {
+        setFocusText(false);
+        return;
+    }
+
+    QInputMethodQueryEvent query(Qt::ImEnabled);
+    qApp->sendEvent(object, &query);
+    setFocusText(query.value(Qt::ImEnabled).toBool());
+}
+
+void OffscreenQmlSurface::setFocusText(bool newFocusText) {
+    if (newFocusText != _focusText) {
+        _focusText = newFocusText;
+        emit focusTextChanged(_focusText);
+    }
+}

libraries/gl/src/gl/OffscreenQmlSurface.h

@@ -30,7 +30,7 @@ class OffscreenQmlRenderThread;
 
 class OffscreenQmlSurface : public QObject {
     Q_OBJECT
-
+    Q_PROPERTY(bool focusText READ isFocusText NOTIFY focusTextChanged)
 public:
     OffscreenQmlSurface();
     virtual ~OffscreenQmlSurface();
@@ -55,6 +55,7 @@ public:
         _mouseTranslator = mouseTranslator;
     }
 
+    bool isFocusText() const { return _focusText; }
     void pause();
     void resume();
     bool isPaused() const;
@@ -70,6 +71,8 @@ public:
 
 signals:
     void textureUpdated(unsigned int texture);
+    void focusObjectChanged(QObject* newFocus);
+    void focusTextChanged(bool focusText);
 
 public slots:
     void requestUpdate();
@@ -78,6 +81,7 @@ public slots:
 
 protected:
     bool filterEnabled(QObject* originalDestination, QEvent* event) const;
+    void setFocusText(bool newFocusText);
 
 private:
     QObject* finishQmlLoad(std::function<void(QQmlContext*, QObject*)> f);
@@ -85,6 +89,7 @@ private:
 
 private slots:
     void updateQuick();
+    void onFocusObjectChanged(QObject* newFocus);
 
 private:
     friend class OffscreenQmlRenderThread;
@@ -97,6 +102,7 @@ private:
     bool _render{ false };
     bool _polish{ true };
     bool _paused{ true };
+    bool _focusText { false };
     uint8_t _maxFps{ 60 };
     MouseTranslator _mouseTranslator{ [](const QPointF& p) { return p.toPoint();  } };
     QWindow* _proxyWindow { nullptr };

plugins/openvr/src/OpenVrDisplayPlugin.cpp

@@ -36,12 +36,14 @@ vec3 _trackedDeviceLinearVelocities[vr::k_unMaxTrackedDeviceCount];
 vec3 _trackedDeviceAngularVelocities[vr::k_unMaxTrackedDeviceCount];
 static mat4 _sensorResetMat;
 static std::array<vr::Hmd_Eye, 2> VR_EYES { { vr::Eye_Left, vr::Eye_Right } };
+bool _openVrDisplayActive { false };
 
 bool OpenVrDisplayPlugin::isSupported() const {
     return openVrSupported();
 }
 
 bool OpenVrDisplayPlugin::internalActivate() {
+    _openVrDisplayActive = true;
     _container->setIsOptionChecked(StandingHMDSensorMode, true);
 
     if (!_system) {
@@ -94,6 +96,7 @@ bool OpenVrDisplayPlugin::internalActivate() {
 
 void OpenVrDisplayPlugin::internalDeactivate() {
     Parent::internalDeactivate();
+    _openVrDisplayActive = false;
     _container->setIsOptionChecked(StandingHMDSensorMode, false);
     if (_system) {
         // Invalidate poses. It's fine if someone else sets these shared values, but we're about to stop updating them, and

plugins/openvr/src/OpenVrHelpers.cpp

@@ -14,9 +14,13 @@
 #include <QtCore/QTimer>
 #include <QtCore/QLoggingCategory>
 #include <QtCore/QProcessEnvironment>
+#include <QtGui/QInputMethodEvent>
+#include <QtQuick/QQuickWindow>
 
 #include <Windows.h>
 
+#include <OffscreenUi.h>
+
 Q_DECLARE_LOGGING_CATEGORY(displayplugins)
 Q_LOGGING_CATEGORY(displayplugins, "hifi.plugins.display")
 
@@ -90,6 +94,121 @@ void releaseOpenVrSystem() {
     }
 }
 
+static char textArray[8192];
+
+static QMetaObject::Connection _focusConnection, _focusTextConnection;
+extern bool _openVrDisplayActive;
+static vr::IVROverlay* _overlay { nullptr };
+static QObject* _keyboardFocusObject { nullptr };
+static QString _existingText;
+static Qt::InputMethodHints _currentHints;
+extern vr::TrackedDevicePose_t _trackedDevicePose[vr::k_unMaxTrackedDeviceCount];
+static bool _keyboardShown { false };
+static const uint32_t SHOW_KEYBOARD_DELAY_MS = 100;
+
+void showOpenVrKeyboard(bool show = true) {
+    if (!_overlay) {
+        return;
+    }
+
+    if (show) {
+        // To avoid flickering the keyboard when a text element is only briefly selected, 
+        // show the keyboard asynchrnously after a very short delay, but only after we check 
+        // that the current focus object is still one that is text enabled
+        QTimer::singleShot(SHOW_KEYBOARD_DELAY_MS, [] {
+            auto offscreenUi = DependencyManager::get<OffscreenUi>();
+            auto currentFocus = offscreenUi->getWindow()->focusObject();
+            QInputMethodQueryEvent query(Qt::ImEnabled | Qt::ImQueryInput | Qt::ImHints);
+            qApp->sendEvent(currentFocus, &query);
+            // Current focus isn't text enabled, bail early.
+            if (!query.value(Qt::ImEnabled).toBool()) {
+                return;
+            }
+            // We're going to show the keyboard now...
+            _keyboardFocusObject = currentFocus;
+            _currentHints = Qt::InputMethodHints(query.value(Qt::ImHints).toUInt());
+            vr::EGamepadTextInputMode inputMode = vr::k_EGamepadTextInputModeNormal;
+            if (_currentHints & Qt::ImhHiddenText) {
+                inputMode = vr::k_EGamepadTextInputModePassword;
+            }
+            vr::EGamepadTextInputLineMode lineMode = vr::k_EGamepadTextInputLineModeSingleLine;
+            if (_currentHints & Qt::ImhMultiLine) {
+                lineMode = vr::k_EGamepadTextInputLineModeMultipleLines;
+            }
+            _existingText = query.value(Qt::ImSurroundingText).toString();
+
+            auto showKeyboardResult = _overlay->ShowKeyboard(inputMode, lineMode, "Keyboard", 1024,
+                _existingText.toLocal8Bit().toStdString().c_str(), false, 0);
+
+            if (vr::VROverlayError_None == showKeyboardResult) {
+                _keyboardShown = true;
+                // Try to position the keyboard slightly below where the user is looking.
+                mat4 headPose = toGlm(_trackedDevicePose[0].mDeviceToAbsoluteTracking);
+                mat4 keyboardTransform = glm::translate(headPose, vec3(0, -0.5, -1));
+                keyboardTransform = keyboardTransform * glm::rotate(mat4(), 3.14159f / 4.0f, vec3(-1, 0, 0));
+                auto keyboardTransformVr = toOpenVr(keyboardTransform);
+                _overlay->SetKeyboardTransformAbsolute(vr::ETrackingUniverseOrigin::TrackingUniverseStanding, &keyboardTransformVr);
+            }
+        });
+    } else {
+        _keyboardFocusObject = nullptr;
+        if (_keyboardShown) {
+            _overlay->HideKeyboard();
+            _keyboardShown = false;
+        }
+    }
+}
+
+void finishOpenVrKeyboardInput() {
+    auto offscreenUi = DependencyManager::get<OffscreenUi>();
+    auto chars = _overlay->GetKeyboardText(textArray, 8192);
+    auto newText = QString(QByteArray(textArray, chars));
+    // TODO modify the new text to match the possible input hints:
+    // ImhDigitsOnly  ImhFormattedNumbersOnly  ImhUppercaseOnly  ImhLowercaseOnly 
+    // ImhDialableCharactersOnly ImhEmailCharactersOnly  ImhUrlCharactersOnly  ImhLatinOnly
+    QInputMethodEvent event(_existingText, QList<QInputMethodEvent::Attribute>());
+    event.setCommitString(newText, 0, _existingText.size());
+    qApp->sendEvent(_keyboardFocusObject, &event);
+    // Simulate an enter press on the top level window to trigger the action
+    if (0 == (_currentHints & Qt::ImhMultiLine)) {
+        qApp->sendEvent(offscreenUi->getWindow(), &QKeyEvent(QEvent::KeyPress, Qt::Key_Return, Qt::KeyboardModifiers(), QString("\n")));
+        qApp->sendEvent(offscreenUi->getWindow(), &QKeyEvent(QEvent::KeyRelease, Qt::Key_Return, Qt::KeyboardModifiers()));
+    }
+}
+
+static const QString DEBUG_FLAG("HIFI_DISABLE_STEAM_VR_KEYBOARD");
+bool disableSteamVrKeyboard = QProcessEnvironment::systemEnvironment().contains(DEBUG_FLAG);
+
+void enableOpenVrKeyboard() {
+    if (disableSteamVrKeyboard) {
+        return;
+    }
+    auto offscreenUi = DependencyManager::get<OffscreenUi>();
+    _overlay = vr::VROverlay();
+
+    _focusConnection = QObject::connect(offscreenUi->getWindow(), &QQuickWindow::focusObjectChanged, [](QObject* object) {
+        if (object != _keyboardFocusObject) {
+            showOpenVrKeyboard(false);
+        }
+    });
+
+    _focusTextConnection = QObject::connect(offscreenUi.data(), &OffscreenUi::focusTextChanged, [](bool focusText) {
+        if (_openVrDisplayActive) {
+            showOpenVrKeyboard(focusText);
+        }
+    });
+}
+
+
+void disableOpenVrKeyboard() {
+    if (disableSteamVrKeyboard) {
+        return;
+    }
+    QObject::disconnect(_focusTextConnection);
+    QObject::disconnect(_focusConnection);
+}
+
+
 void handleOpenVrEvents() {
     if (!activeHmd) {
         return;
@@ -107,6 +226,10 @@ void handleOpenVrEvents() {
                 activeHmd->AcknowledgeQuit_Exiting();
                 break;
 
+            case vr::VREvent_KeyboardDone: 
+                finishOpenVrKeyboardInput();
+                break;
+
             default:
                 break;
         }
@@ -114,3 +237,4 @@ void handleOpenVrEvents() {
     }
 
 }
+

plugins/openvr/src/OpenVrHelpers.h

@@ -18,6 +18,9 @@ vr::IVRSystem* acquireOpenVrSystem();
 void releaseOpenVrSystem();
 void handleOpenVrEvents();
 bool openVrQuitRequested();
+void enableOpenVrKeyboard();
+void disableOpenVrKeyboard();
+
 
 template<typename F>
 void openvr_for_each_eye(F f) {
@@ -41,3 +44,13 @@ inline mat4 toGlm(const vr::HmdMatrix34_t& m) {
         m.m[0][3], m.m[1][3], m.m[2][3], 1.0f);
     return result;
 }
+
+inline vr::HmdMatrix34_t toOpenVr(const mat4& m) {
+    vr::HmdMatrix34_t result;
+    for (uint8_t i = 0; i < 3; ++i) {
+        for (uint8_t j = 0; j < 4; ++j) {
+            result.m[i][j] = m[j][i];
+        }
+    }
+    return result;
+}

plugins/openvr/src/ViveControllerManager.cpp

@@ -11,8 +11,6 @@
 
 #include "ViveControllerManager.h"
 
-#include <QtCore/QProcessEnvironment>
-
 #include <PerfStat.h>
 #include <PathUtils.h>
 #include <GeometryCache.h>
@@ -22,6 +20,7 @@
 #include <NumericalConstants.h>
 #include <plugins/PluginContainer.h>
 #include <UserActivityLogger.h>
+#include <OffscreenUi.h>
 
 #include <controllers/UserInputMapper.h>
 
@@ -68,6 +67,8 @@ bool ViveControllerManager::activate() {
     }
     Q_ASSERT(_system);
 
+    enableOpenVrKeyboard();
+
     // OpenVR provides 3d mesh representations of the controllers
     // Disabled controller rendering code
     /*
@@ -132,6 +133,8 @@ bool ViveControllerManager::activate() {
 void ViveControllerManager::deactivate() {
     InputPlugin::deactivate();
 
+    disableOpenVrKeyboard();
+
     _container->removeMenuItem(MENU_NAME, RENDER_CONTROLLERS);
     _container->removeMenu(MENU_PATH);