Merge pull request #14875 from jherico/feature/quest_custom_plugins

Case 20882: Support custom display / input plugin lists and small tweaks

CMakeLists.txt

@@ -81,6 +81,25 @@ if (ANDROID)
   set(GLES_OPTION ON)
   set(PLATFORM_QT_COMPONENTS AndroidExtras WebView)
   add_definitions(-DHIFI_ANDROID_APP=\"${HIFI_ANDROID_APP}\")
+  if (
+      (${HIFI_ANDROID_APP} STREQUAL "questInterface") OR 
+      (${HIFI_ANDROID_APP} STREQUAL "questFramePlayer") OR
+      (${HIFI_ANDROID_APP} STREQUAL "framePlayer") 
+    )
+    # We know the quest hardware has this extension, so we can force the use of instanced stereo
+    add_definitions(-DHAVE_EXT_clip_cull_distance)
+    # We can also use multiview stereo techniques
+    add_definitions(-DHAVE_OVR_multiview2)
+    add_definitions(-DHAVE_OVR_multiview)
+    # We can also use our own foveated textures
+    add_definitions(-DHAVE_QCOM_texture_foveated)
+
+    # if set, the application itself or some library it depends on MUST implement 
+    # `DisplayPluginList getDisplayPlugins()` and `InputPluginList getInputPlugins()`
+    add_definitions(-DCUSTOM_INPUT_PLUGINS) 
+    add_definitions(-DCUSTOM_DISPLAY_PLUGINS) 
+    set(PLATFORM_PLUGIN_LIBRARIES oculusMobile oculusMobilePlugin)
+  endif()
 else ()
   set(PLATFORM_QT_COMPONENTS WebEngine Xml)
 endif ()

android/apps/framePlayer/build.gradle

@@ -1,15 +1,8 @@
+import com.android.builder.core.BuilderConstants
+
 apply plugin: 'com.android.application'
 
 android {
-    signingConfigs {
-        release {
-            storeFile project.hasProperty("HIFI_ANDROID_KEYSTORE") ? file(HIFI_ANDROID_KEYSTORE) : null
-            storePassword project.hasProperty("HIFI_ANDROID_KEYSTORE_PASSWORD") ? HIFI_ANDROID_KEYSTORE_PASSWORD : ''
-            keyAlias project.hasProperty("HIFI_ANDROID_KEY_ALIAS") ? HIFI_ANDROID_KEY_ALIAS : ''
-            keyPassword project.hasProperty("HIFI_ANDROID_KEY_PASSWORD") ? HIFI_ANDROID_KEY_PASSWORD : ''
-        }
-    }
-
     compileSdkVersion 28
     defaultConfig {
         applicationId "io.highfidelity.frameplayer"
@@ -32,19 +25,17 @@ android {
         targetCompatibility JavaVersion.VERSION_1_8
     }
 
-    buildTypes {
+    externalNativeBuild.cmake.path '../../../CMakeLists.txt'
-        release {
+
-            minifyEnabled false
+    variantFilter { variant ->
-            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        def build = variant.buildType.name
-            signingConfig signingConfigs.release
+        if (build == BuilderConstants.RELEASE) {
+            variant.setIgnore(true)
         }
     }
-
-    externalNativeBuild.cmake.path '../../../CMakeLists.txt'
 }
 
 dependencies {
     implementation fileTree(include: ['*.jar'], dir: '../../libraries/qt/libs')
-    //implementation project(':oculus')
     implementation project(':qt')
 }

android/apps/questFramePlayer/build.gradle

@@ -1,15 +1,8 @@
+import com.android.builder.core.BuilderConstants
+
 apply plugin: 'com.android.application'
 
 android {
-    signingConfigs {
-        release {
-            storeFile project.hasProperty("HIFI_ANDROID_KEYSTORE") ? file(HIFI_ANDROID_KEYSTORE) : null
-            storePassword project.hasProperty("HIFI_ANDROID_KEYSTORE_PASSWORD") ? HIFI_ANDROID_KEYSTORE_PASSWORD : ''
-            keyAlias project.hasProperty("HIFI_ANDROID_KEY_ALIAS") ? HIFI_ANDROID_KEY_ALIAS : ''
-            keyPassword project.hasProperty("HIFI_ANDROID_KEY_PASSWORD") ? HIFI_ANDROID_KEY_PASSWORD : ''
-        }
-    }
-
     compileSdkVersion 28
     defaultConfig {
         applicationId "io.highfidelity.frameplayer"
@@ -33,15 +26,14 @@ android {
         targetCompatibility JavaVersion.VERSION_1_8
     }
 
-    buildTypes {
+    externalNativeBuild.cmake.path '../../../CMakeLists.txt'
-        release {
+
-            minifyEnabled false
+    variantFilter { variant ->
-            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        def build = variant.buildType.name
-            signingConfig signingConfigs.release
+        if (build == BuilderConstants.RELEASE) {
+            variant.setIgnore(true)
         }
     }
-
-    externalNativeBuild.cmake.path '../../../CMakeLists.txt'
 }
 
 dependencies {

android/libraries/qt/src/main/java/org/qtproject/qt5/android/bindings/QtActivity.java

@@ -650,6 +650,8 @@ public class QtActivity extends Activity {
         if (!keepInterfaceRunning) {
             QtApplication.invokeDelegate();
         }
+        QtNative.terminateQt();
+        QtNative.setActivity(null,null);
     }
 
     //---------------------------------------------------------------------------

interface/CMakeLists.txt

@@ -211,7 +211,10 @@ link_hifi_libraries(
   render-utils entities-renderer avatars-renderer ui qml auto-updater midi
   controllers plugins image trackers
   ui-plugins display-plugins input-plugins
+  # Platform specific GL libraries
   ${PLATFORM_GL_BACKEND}
+  # Plaform specific input & display plugin libraries
+  ${PLATFORM_PLUGIN_LIBRARIES}
   shaders
 )
 

interface/src/Application.cpp

@@ -603,8 +603,9 @@ void messageHandler(QtMsgType type, const QMessageLogContext& context, const QSt
                 __android_log_write(ANDROID_LOG_FATAL,"Interface",local);
                 abort();
         }
-#endif
+#else
         qApp->getLogger()->addMessage(qPrintable(logMessage));
+#endif
     }
 }
 
@@ -989,7 +990,9 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     QApplication(argc, argv),
     _window(new MainWindow(desktop())),
     _sessionRunTimer(startupTimer),
+#ifndef Q_OS_ANDROID
     _logger(new FileLogger(this)),
+#endif
     _previousSessionCrashed(setupEssentials(argc, argv, runningMarkerExisted)),
     _entitySimulation(new PhysicalEntitySimulation()),
     _physicsEngine(new PhysicsEngine(Vectors::ZERO)),
@@ -1123,7 +1126,9 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     }
     auto accountManager = DependencyManager::get<AccountManager>();
 
+#ifndef Q_OS_ANDROID
     _logger->setSessionID(accountManager->getSessionID());
+#endif
 
     setCrashAnnotation("metaverse_session_id", accountManager->getSessionID().toString().toStdString());
     setCrashAnnotation("main_thread_id", std::to_string((size_t)QThread::currentThreadId()));
@@ -4963,7 +4968,11 @@ void Application::idle() {
     // Normally we check PipelineWarnings, but since idle will often take more than 10ms we only show these idle timing
     // details if we're in ExtraDebugging mode. However, the ::update() and its subcomponents will show their timing
     // details normally.
+#ifdef Q_OS_ANDROID
+    bool showWarnings = false;
+#else
     bool showWarnings = getLogger()->extraDebugging();
+#endif
     PerformanceWarning warn(showWarnings, "idle()");
 
     {
@@ -8306,7 +8315,7 @@ void Application::toggleLogDialog() {
         bool keepOnTop =_keepLogWindowOnTop.get();
 #ifdef Q_OS_WIN
         _logDialog = new LogDialog(keepOnTop ? qApp->getWindow() : nullptr, getLogger());
-#else
+#elif !defined(Q_OS_ANDROID)
         _logDialog = new LogDialog(nullptr, getLogger());
 
         if (keepOnTop) {

interface/src/Application.h

@@ -251,7 +251,9 @@ public:
 
     void setActiveDisplayPlugin(const QString& pluginName);
 
+#ifndef Q_OS_ANDROID
     FileLogger* getLogger() const { return _logger; }
+#endif
 
     float getRenderResolutionScale() const;
 
@@ -599,7 +601,9 @@ private:
 
     bool _aboutToQuit { false };
 
+#ifndef Q_OS_ANDROID
     FileLogger* _logger { nullptr };
+#endif
 
     bool _previousSessionCrashed;
 

interface/src/octree/OctreePacketProcessor.cpp

@@ -38,11 +38,13 @@ void OctreePacketProcessor::processPacket(QSharedPointer<ReceivedMessage> messag
     PerformanceWarning warn(Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings),
                             "OctreePacketProcessor::processPacket()");
 
+#ifndef Q_OS_ANDROID
     const int WAY_BEHIND = 300;
 
     if (packetsToProcessCount() > WAY_BEHIND && qApp->getLogger()->extraDebugging()) {
         qDebug("OctreePacketProcessor::processPacket() packets to process=%d", packetsToProcessCount());
     }
+#endif
     
     bool wasStatsPacket = false;
 

libraries/display-plugins/src/display-plugins/DisplayPlugin.cpp

@@ -20,6 +20,7 @@ const QString& DisplayPlugin::MENU_PATH() {
     return value;
 }
 
+#if !defined(CUSTOM_DISPLAY_PLUGINS)
 // TODO migrate to a DLL model where plugins are discovered and loaded at runtime by the PluginManager class
 DisplayPluginList getDisplayPlugins() {
     DisplayPlugin* PLUGIN_POOL[] = {
@@ -49,3 +50,4 @@ DisplayPluginList getDisplayPlugins() {
     }
     return result;
 }
+#endif

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp

@@ -720,6 +720,8 @@ void OpenGLDisplayPlugin::present() {
         }
 
         gpu::Backend::freeGPUMemSize.set(gpu::gl::getFreeDedicatedMemory());
+    } else {
+        internalPresent();
     }
     _movingAveragePresent.addSample((float)(usecTimestampNow() - startPresent));
 }

libraries/display-plugins/src/display-plugins/hmd/DebugHmdDisplayPlugin.cpp

@@ -51,15 +51,15 @@ bool DebugHmdDisplayPlugin::internalActivate() {
     }, true, _isAutoRotateEnabled);
 
     _ipd = 0.0327499993f * 2.0f;
-    // Would be nice to know why the left and right projection matrices are slightly dissymetrical
+    // Quest 
-    _eyeProjections[0][0] = vec4{ 0.759056330, 0.000000000, 0.000000000, 0.000000000 };
+    _eyeProjections[0][0] = vec4{ 0.91729, 0.0, -0.17407, 0.0  };
-    _eyeProjections[0][1] = vec4{ 0.000000000, 0.682773232, 0.000000000, 0.000000000 };
+    _eyeProjections[0][1] = vec4{ 0.0, 0.083354, -0.106141, 0.0 };
-    _eyeProjections[0][2] = vec4{ -0.0580431037, -0.00619550655, -1.00000489, -1.00000000 };
+    _eyeProjections[0][2] = vec4{ 0.0, 0.0, -1.0, -0.2 };
-    _eyeProjections[0][3] = vec4{ 0.000000000, 0.000000000, -0.0800003856, 0.000000000 };
+    _eyeProjections[0][3] = vec4{ 0.0, 0.0, -1.0, 0.0 };
-    _eyeProjections[1][0] = vec4{ 0.752847493, 0.000000000, 0.000000000, 0.000000000 };
+    _eyeProjections[1][0] = vec4{ 0.91729, 0.0, 0.17407, 0.0 };
-    _eyeProjections[1][1] = vec4{ 0.000000000, 0.678060353, 0.000000000, 0.000000000 };
+    _eyeProjections[1][1] = vec4{ 0.0, 0.083354, -0.106141, 0.0 };
-    _eyeProjections[1][2] = vec4{ 0.0578232110, -0.00669418881, -1.00000489, -1.000000000 };
+    _eyeProjections[1][2] = vec4{ 0.0, 0.0, -1.0, -0.2 };
-    _eyeProjections[1][3] = vec4{ 0.000000000, 0.000000000, -0.0800003856, 0.000000000 };
+    _eyeProjections[1][3] = vec4{ 0.0, 0.0, -1.0, 0.0 };
     // No need to do so here as this will done in Parent::internalActivate
     //_eyeInverseProjections[0] = glm::inverse(_eyeProjections[0]);
     //_eyeInverseProjections[1] = glm::inverse(_eyeProjections[1]);

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.h

@@ -81,7 +81,6 @@ protected:
         mat4 presentPose;
         double sensorSampleTime { 0 };
         double predictedDisplayTime { 0 };
-        mat3 presentReprojection;
     };
 
     QMap<uint32_t, FrameInfo> _frameInfos;

libraries/gpu-gl-common/src/gpu/gl/GLBackend.cpp

@@ -418,6 +418,10 @@ public:
 #endif
 };
 
+#if defined(GPU_STEREO_DRAWCALL_INSTANCED) && !defined(GL_CLIP_DISTANCE0)
+#define GL_CLIP_DISTANCE0 GL_CLIP_DISTANCE0_EXT
+#endif
+
 #define GL_PROFILE_RANGE(category, name) \
     PROFILE_RANGE(category, name); \
     GlDuration glProfileRangeThis(name);

libraries/gpu-gl-common/src/gpu/gl/GLBackend.h

@@ -32,7 +32,7 @@
 
 // Different versions for the stereo drawcall
 // Current preferred is  "instanced" which draw the shape twice but instanced and rely on clipping plane to draw left/right side only
-#if defined(USE_GLES)
+#if defined(USE_GLES) && !defined(HAVE_EXT_clip_cull_distance)
 #define GPU_STEREO_TECHNIQUE_DOUBLED_SIMPLE
 #else
 //#define GPU_STEREO_TECHNIQUE_DOUBLED_SMARTER

libraries/input-plugins/src/input-plugins/InputPlugin.cpp

@@ -16,6 +16,7 @@
 #include "TouchscreenDevice.h"
 #include "TouchscreenVirtualPadDevice.h"
 
+#if !defined(CUSTOM_INPUT_PLUGINS)
 // TODO migrate to a DLL model where plugins are discovered and loaded at runtime by the PluginManager class
 InputPluginList getInputPlugins() {
     InputPlugin* PLUGIN_POOL[] = {
@@ -37,6 +38,7 @@ InputPluginList getInputPlugins() {
     }
     return result;
 }
+#endif
 
 void saveInputPluginSettings(const InputPluginList& plugins) {
     foreach (auto inputPlugin, plugins) {

libraries/oculusMobilePlugin/src/OculusMobileControllerManager.cpp

@@ -171,6 +171,8 @@ private:
     void handleHeadPose(float deltaTime, const controller::InputCalibrationData& inputCalibrationData,
                         const ovrRigidBodyPosef& headPose);
 
+    void reconnectTouchControllers(ovrMobile* session);
+
     // perform an action when the TouchDevice mutex is acquired.
     using Locker = std::unique_lock<std::recursive_mutex>;
 
@@ -637,7 +639,6 @@ controller::Input::NamedVector OculusMobileInputDevice::getAvailableInputs() con
         makePair(RIGHT_THUMB_UP, "RightThumbUp"),
         makePair(LEFT_INDEX_POINT, "LeftIndexPoint"),
         makePair(RIGHT_INDEX_POINT, "RightIndexPoint"),
-
         makePair(BACK, "LeftApplicationMenu"),
         makePair(START, "RightApplicationMenu"),
     };
@@ -665,8 +666,40 @@ OculusMobileInputDevice::OculusMobileInputDevice(ovrMobile* session, const std::
 
 void OculusMobileInputDevice::updateHands(ovrMobile* session) {
     _headTracking = vrapi_GetPredictedTracking2(session, 0.0);
+
+    bool touchControllerNotConnected = false;
     for (auto& hand : _hands) {
         hand.update(session);
+
+        if (hand.stateResult < 0 || hand.trackingResult < 0) {
+            touchControllerNotConnected = true;
+        }
+    }
+
+    if (touchControllerNotConnected) {
+        reconnectTouchControllers(session);
+    }
+}
+
+void OculusMobileInputDevice::reconnectTouchControllers(ovrMobile* session) {
+    uint32_t deviceIndex { 0 };
+    ovrInputCapabilityHeader capsHeader;
+    while (vrapi_EnumerateInputDevices(session, deviceIndex, &capsHeader) >= 0) {
+        if (capsHeader.Type == ovrControllerType_TrackedRemote) {
+            ovrInputTrackedRemoteCapabilities caps;
+            caps.Header = capsHeader;
+            vrapi_GetInputDeviceCapabilities(session, &caps.Header);
+
+            if (caps.ControllerCapabilities & ovrControllerCaps_LeftHand || caps.ControllerCapabilities & ovrControllerCaps_RightHand) {
+                size_t handIndex = caps.ControllerCapabilities & ovrControllerCaps_LeftHand ? 0 : 1;
+                HandData& handData = _hands[handIndex];
+                handData.state.Header.ControllerType = ovrControllerType_TrackedRemote;
+                handData.valid = true;
+                handData.caps = caps;
+                handData.update(session);
+            }
+        }
+        ++deviceIndex;
     }
 }
 
@@ -691,4 +724,4 @@ InputPluginList getInputPlugins() {
         }
     }
     return result;
-}
+}

libraries/oculusMobilePlugin/src/OculusMobileDisplayPlugin.cpp

@@ -63,7 +63,7 @@ bool OculusMobileDisplayPlugin::internalActivate() {
 
     withOvrJava([&](const ovrJava* java){
         _renderTargetSize = glm::uvec2{
-            vrapi_GetSystemPropertyInt(java, VRAPI_SYS_PROP_SUGGESTED_EYE_TEXTURE_WIDTH),
+            2 * vrapi_GetSystemPropertyInt(java, VRAPI_SYS_PROP_SUGGESTED_EYE_TEXTURE_WIDTH),
             vrapi_GetSystemPropertyInt(java, VRAPI_SYS_PROP_SUGGESTED_EYE_TEXTURE_HEIGHT),
         };
     });

libraries/qml/src/qml/OffscreenSurface.cpp

@@ -14,6 +14,7 @@
 #include <QtQml/QtQml>
 #include <QtQml/QQmlEngine>
 #include <QtQml/QQmlComponent>
+#include <QtQml/QQmlFileSelector>
 #include <QtQuick/QQuickItem>
 #include <QtQuick/QQuickWindow>
 #include <QtQuick/QQuickRenderControl>
@@ -43,7 +44,21 @@ static QSize clampSize(const QSize& qsize, uint32_t maxDimension) {
 const QmlContextObjectCallback OffscreenSurface::DEFAULT_CONTEXT_OBJECT_CALLBACK = [](QQmlContext*, QQuickItem*) {};
 const QmlContextCallback OffscreenSurface::DEFAULT_CONTEXT_CALLBACK = [](QQmlContext*) {};
 
+QQmlFileSelector* OffscreenSurface::getFileSelector() {
+    auto context = getSurfaceContext();
+    if (!context) {
+        return nullptr;
+    }
+    auto engine = context->engine();
+    if (!engine) {
+        return nullptr;
+    }
+
+    return QQmlFileSelector::get(engine);
+}
+
 void OffscreenSurface::initializeEngine(QQmlEngine* engine) {
+    new QQmlFileSelector(engine);
 }
 
 using namespace hifi::qml::impl;

libraries/qml/src/qml/OffscreenSurface.h

@@ -30,6 +30,7 @@ class QQmlComponent;
 class QQuickWindow;
 class QQuickItem;
 class OffscreenQmlSharedObject;
+class QQmlFileSelector;
 
 namespace hifi { namespace qml {
 
@@ -72,6 +73,7 @@ public:
     QQuickWindow* getWindow();
     QObject* getEventHandler();
     QQmlContext* getSurfaceContext();
+    QQmlFileSelector* getFileSelector();
 
     // Checks to see if a new texture is available.  If one is, the function returns true and
     // textureAndFence will be populated with the texture ID and a fence which will be signalled

libraries/ui/src/ui/OffscreenQmlSurface.cpp

@@ -238,7 +238,10 @@ void OffscreenQmlSurface::clearFocusItem() {
 
 void OffscreenQmlSurface::initializeEngine(QQmlEngine* engine) {
     Parent::initializeEngine(engine);
-    QQmlFileSelector* fileSelector = new QQmlFileSelector(engine);
+    auto fileSelector = QQmlFileSelector::get(engine);
+    if (!fileSelector) {
+        fileSelector = new QQmlFileSelector(engine);
+    }
     fileSelector->setExtraSelectors(FileUtils::getFileSelectors());
 
     static std::once_flag once;

plugins/oculus/src/OculusDisplayPlugin.cpp

@@ -138,6 +138,10 @@ void OculusDisplayPlugin::hmdPresent() {
         return;
     }
 
+    if (!_currentFrame) {
+        return;
+    }
+
     PROFILE_RANGE_EX(render, __FUNCTION__, 0xff00ff00, (uint64_t)_currentFrame->frameIndex)
 
     {

tools/gpu-frame-player/CMakeLists.txt

@@ -7,7 +7,7 @@ setup_hifi_project(Gui Widgets)
 
 # link in the shared libraries
 link_hifi_libraries(
-    shared ktx shaders gpu 
+    shared ktx shaders gpu
 #    vk gpu-vk 
     gl ${PLATFORM_GL_BACKEND}
 )