Merge pull request #9173 from sethalves/tablet-ui

Tablet ui

interface/src/Application.cpp

@@ -1018,7 +1018,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
                     toggleMenuUnderReticle();
                 }
             } else if (action == controller::toInt(controller::Action::CONTEXT_MENU)) {
-                toggleMenuUnderReticle();
+                toggleTabletUI();
             } else if (action == controller::toInt(controller::Action::RETICLE_X)) {
                 auto oldPos = getApplicationCompositor().getReticlePosition();
                 getApplicationCompositor().setReticlePosition({ oldPos.x + state, oldPos.y });
@@ -1484,6 +1484,11 @@ void Application::toggleMenuUnderReticle() const {
     offscreenUi->toggleMenu(QPoint(reticlePosition.x - X_LEFT_SHIFT, reticlePosition.y));
 }
 
+void Application::toggleTabletUI() const {
+    auto HMD = DependencyManager::get<HMDScriptingInterface>();
+    HMD->toggleShouldShowTablet();
+}
+
 void Application::checkChangeCursor() {
     QMutexLocker locker(&_changeCursorLock);
     if (_cursorNeedsChanging) {

interface/src/Application.h

@@ -455,6 +455,7 @@ private:
 
     void maybeToggleMenuVisible(QMouseEvent* event) const;
     void toggleMenuUnderReticle() const;
+    void toggleTabletUI() const;
 
     MainWindow* _window;
     QElapsedTimer& _sessionRunTimer;

interface/src/scripting/HMDScriptingInterface.cpp

@@ -18,6 +18,7 @@
 #include <display-plugins/CompositorHelper.h>
 #include <OffscreenUi.h>
 #include <plugins/PluginUtils.h>
+#include <QUuid>
 
 #include "Application.h"
 

interface/src/scripting/HMDScriptingInterface.h

@@ -28,6 +28,8 @@ class HMDScriptingInterface : public AbstractHMDScriptingInterface, public Depen
     Q_PROPERTY(glm::vec3 position READ getPosition)
     Q_PROPERTY(glm::quat orientation READ getOrientation)
     Q_PROPERTY(bool mounted READ isMounted)
+    Q_PROPERTY(bool showTablet READ getShouldShowTablet)
+    Q_PROPERTY(QUuid tabletID READ getCurrentTableUIID WRITE setCurrentTabletUIID)
 
 public:
     Q_INVOKABLE glm::vec3 calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction) const;
@@ -52,11 +54,11 @@ public:
     Q_INVOKABLE void disableExtraLaser() const;
 
 
-    /// Suppress the activation of any on-screen keyboard so that a script operation will 
+    /// Suppress the activation of any on-screen keyboard so that a script operation will
     /// not be interrupted by a keyboard popup
     /// Returns false if there is already an active keyboard displayed.
     /// Clients should re-enable the keyboard when the operation is complete and ensure
-    /// that they balance any call to suppressKeyboard() that returns true with a corresponding 
+    /// that they balance any call to suppressKeyboard() that returns true with a corresponding
     /// call to unsuppressKeyboard() within a reasonable amount of time
     Q_INVOKABLE bool suppressKeyboard();
 
@@ -79,10 +81,19 @@ public:
 
     bool isMounted() const;
 
+    void toggleShouldShowTablet() { _showTablet = !_showTablet; }
+    bool getShouldShowTablet() const { return _showTablet; }
+
+    void setCurrentTabletUIID(QUuid tabletID) { _tabletUIID = tabletID; }
+    QUuid getCurrentTableUIID() { return _tabletUIID; }
+
 private:
+    bool _showTablet { false };
+    QUuid _tabletUIID; // this is the entityID of the WebEntity which is part of (a child of) the tablet-ui.
+
     // Get the position of the HMD
     glm::vec3 getPosition() const;
-    
+
     // Get the orientation of the HMD
     glm::quat getOrientation() const;
 

libraries/entities-renderer/src/RenderableWebEntityItem.cpp

@@ -18,7 +18,6 @@
 
 #include <GeometryCache.h>
 #include <PerfStat.h>
-#include <gl/OffscreenQmlSurface.h>
 #include <AbstractViewStateInterface.h>
 #include <GLMHelpers.h>
 #include <PathUtils.h>
@@ -46,8 +45,6 @@ EntityItemPointer RenderableWebEntityItem::factory(const EntityItemID& entityID,
 
 RenderableWebEntityItem::RenderableWebEntityItem(const EntityItemID& entityItemID) :
     WebEntityItem(entityItemID) {
-    qDebug() << "Created web entity " << getID();
-
     _touchDevice.setCapabilities(QTouchDevice::Position);
     _touchDevice.setType(QTouchDevice::TouchScreen);
     _touchDevice.setName("RenderableWebEntityItemTouchDevice");
@@ -57,7 +54,6 @@ RenderableWebEntityItem::RenderableWebEntityItem(const EntityItemID& entityItemI
 
 RenderableWebEntityItem::~RenderableWebEntityItem() {
     destroyWebSurface();
-    qDebug() << "Destroyed web entity " << getID();
     auto geometryCache = DependencyManager::get<GeometryCache>();
     if (geometryCache) {
         geometryCache->releaseID(_geometryId);
@@ -78,19 +74,18 @@ bool RenderableWebEntityItem::buildWebSurface(QSharedPointer<EntityTreeRenderer>
         QString createGlobalEventBridgeStr = QTextStream(&createGlobalEventBridgeFile).readAll();
 
         // concatenate these js files
-        javaScriptToInject = webChannelStr + createGlobalEventBridgeStr;
+        _javaScriptToInject = webChannelStr + createGlobalEventBridgeStr;
     } else {
         qCWarning(entitiesrenderer) << "unable to find qwebchannel.js or createGlobalEventBridge.js";
     }
 
     // Save the original GL context, because creating a QML surface will create a new context
-    QOpenGLContext * currentContext = QOpenGLContext::currentContext();
+    QOpenGLContext* currentContext = QOpenGLContext::currentContext();
     if (!currentContext) {
         return false;
     }
 
     ++_currentWebCount;
-    qDebug() << "Building web surface: " << getID() << ", #" << _currentWebCount << ", url = " << _sourceUrl;
 
     QSurface * currentSurface = currentContext->surface();
 
@@ -113,27 +108,15 @@ bool RenderableWebEntityItem::buildWebSurface(QSharedPointer<EntityTreeRenderer>
 
     // The lifetime of the QML surface MUST be managed by the main thread
     // Additionally, we MUST use local variables copied by value, rather than
-    // member variables, since they would implicitly refer to a this that 
+    // member variables, since they would implicitly refer to a this that
     // is no longer valid
     _webSurface->create(currentContext);
 
-#define AJT_TABLET_HACK
+    loadSourceURL();
 
-#ifdef AJT_TABLET_HACK
-    _webSurface->setBaseUrl(QUrl::fromLocalFile(PathUtils::resourcesPath() + "qml/hifi/tablet/"));
-    _webSurface->load("Tablet.qml", [&](QQmlContext* context, QObject* obj) {
-        ;
-    });
-    _webSurface->resume();
-#else
-    _webSurface->setBaseUrl(QUrl::fromLocalFile(PathUtils::resourcesPath() + "/qml/controls/"));
-    _webSurface->load("WebView.qml", [&](QQmlContext* context, QObject* obj) {
-        context->setContextProperty("eventBridgeJavaScriptToInject", QVariant(javaScriptToInject));
-    });
     _webSurface->resume();
     _webSurface->getRootItem()->setProperty("url", _sourceUrl);
     _webSurface->getRootContext()->setContextProperty("desktop", QVariant());
-#endif
     // FIXME - Keyboard HMD only: Possibly add "HMDinfo" object to context for WebView.qml.
 
     // forward web events to EntityScriptingInterface
@@ -167,7 +150,8 @@ bool RenderableWebEntityItem::buildWebSurface(QSharedPointer<EntityTreeRenderer>
             point.setPos(windowPoint);
             QList<QTouchEvent::TouchPoint> touchPoints;
             touchPoints.push_back(point);
-            QTouchEvent* touchEvent = new QTouchEvent(QEvent::TouchEnd, nullptr, Qt::NoModifier, Qt::TouchPointReleased, touchPoints);
+            QTouchEvent* touchEvent = new QTouchEvent(QEvent::TouchEnd, nullptr,
+                                                      Qt::NoModifier, Qt::TouchPointReleased, touchPoints);
             touchEvent->setWindow(_webSurface->getWindow());
             touchEvent->setDevice(&_touchDevice);
             touchEvent->setTarget(_webSurface->getRootItem());
@@ -256,13 +240,35 @@ void RenderableWebEntityItem::render(RenderArgs* args) {
     DependencyManager::get<GeometryCache>()->renderQuad(batch, topLeft, bottomRight, texMin, texMax, glm::vec4(1.0f, 1.0f, 1.0f, fadeRatio), _geometryId);
 }
 
+void RenderableWebEntityItem::loadSourceURL() {
+    QUrl sourceUrl(_sourceUrl);
+    if (sourceUrl.scheme() == "http" || sourceUrl.scheme() == "https" ||
+        _sourceUrl.toLower().endsWith(".htm") || _sourceUrl.toLower().endsWith(".html")) {
+        _contentType = htmlContent;
+        _webSurface->setBaseUrl(QUrl::fromLocalFile(PathUtils::resourcesPath() + "qml/controls/"));
+        _webSurface->load("WebView.qml", [&](QQmlContext* context, QObject* obj) {
+            context->setContextProperty("eventBridgeJavaScriptToInject", QVariant(_javaScriptToInject));
+        });
+        _webSurface->getRootItem()->setProperty("url", _sourceUrl);
+        _webSurface->getRootContext()->setContextProperty("desktop", QVariant());
+
+    } else {
+        _contentType = qmlContent;
+        _webSurface->setBaseUrl(QUrl::fromLocalFile(PathUtils::resourcesPath()));
+        _webSurface->load(_sourceUrl, [&](QQmlContext* context, QObject* obj) { });
+    }
+}
+
+
 void RenderableWebEntityItem::setSourceUrl(const QString& value) {
     if (_sourceUrl != value) {
-        qDebug() << "Setting web entity source URL to " << value;
         _sourceUrl = value;
         if (_webSurface) {
             AbstractViewStateInterface::instance()->postLambdaEvent([this] {
-                _webSurface->getRootItem()->setProperty("url", _sourceUrl);
+                loadSourceURL();
+                if (_contentType == htmlContent) {
+                    _webSurface->getRootItem()->setProperty("url", _sourceUrl);
+                }
             });
         }
     }
@@ -368,8 +374,6 @@ void RenderableWebEntityItem::destroyWebSurface() {
         QObject::disconnect(_hoverLeaveConnection);
         _hoverLeaveConnection = QMetaObject::Connection();
         _webSurface.reset();
-
-        qDebug() << "Delete web surface: " << getID() << ", #" << _currentWebCount << ", url = " << _sourceUrl;
     }
 }
 
@@ -385,6 +389,13 @@ bool RenderableWebEntityItem::isTransparent() {
     return fadeRatio < OPAQUE_ALPHA_THRESHOLD;
 }
 
+QObject* RenderableWebEntityItem::getRootItem() {
+    if (_webSurface) {
+        return dynamic_cast<QObject*>(_webSurface->getRootItem());
+    }
+    return nullptr;
+}
+
 void RenderableWebEntityItem::emitScriptEvent(const QVariant& message) {
     if (_webSurface) {
         _webSurface->emitScriptEvent(message);

libraries/entities-renderer/src/RenderableWebEntityItem.h

@@ -16,6 +16,7 @@
 #include <gl/OffscreenQmlSurface.h>
 
 #include <WebEntityItem.h>
+#include <gl/OffscreenQmlSurface.h>
 
 #include "RenderableEntityItem.h"
 
@@ -33,6 +34,7 @@ public:
     ~RenderableWebEntityItem();
 
     virtual void render(RenderArgs* args) override;
+    void loadSourceURL();
     virtual void setSourceUrl(const QString& value) override;
 
     virtual bool wantsHandControllerPointerEvents() const override { return true; }
@@ -51,6 +53,10 @@ public:
 
     virtual bool isTransparent() override;
 
+ public:
+
+    virtual QObject* getRootItem() override;
+
 private:
     bool buildWebSurface(QSharedPointer<EntityTreeRenderer> renderer);
     void destroyWebSurface();
@@ -69,6 +75,14 @@ private:
     QMetaObject::Connection _mouseReleaseConnection;
     QMetaObject::Connection _mouseMoveConnection;
     QMetaObject::Connection _hoverLeaveConnection;
+
+    QString _javaScriptToInject;
+
+    enum contentType {
+        htmlContent,
+        qmlContent
+    };
+    contentType _contentType;
     int _geometryId { 0 };
 };
 

libraries/entities/src/EntityScriptingInterface.cpp

@@ -25,7 +25,7 @@
 #include "QVariantGLM.h"
 #include "SimulationOwner.h"
 #include "ZoneEntityItem.h"
-
+#include "WebEntityItem.h"
 
 EntityScriptingInterface::EntityScriptingInterface(bool bidOnSimulationOwnership) :
     _entityTree(NULL),
@@ -1453,3 +1453,12 @@ void EntityScriptingInterface::setCostMultiplier(float value) {
     costMultiplier = value;
 }
 
+QObject* EntityScriptingInterface::getWebViewRoot(const QUuid& entityID) {
+    if (auto entity = checkForTreeEntityAndTypeMatch(entityID, EntityTypes::Web)) {
+        auto webEntity = std::dynamic_pointer_cast<WebEntityItem>(entity);
+        QObject* root = webEntity->getRootItem();
+        return root;
+    } else {
+        return nullptr;
+    }
+}

libraries/entities/src/EntityScriptingInterface.h

@@ -276,6 +276,8 @@ public slots:
 
     Q_INVOKABLE void emitScriptEvent(const EntityItemID& entityID, const QVariant& message);
 
+    Q_INVOKABLE QObject* getWebViewRoot(const QUuid& entityID);
+
 signals:
     void collisionWithEntity(const EntityItemID& idA, const EntityItemID& idB, const Collision& collision);
 

libraries/entities/src/WebEntityItem.h

@@ -59,6 +59,8 @@ public:
     void setDPI(uint16_t value);
     uint16_t getDPI() const;
 
+    virtual QObject* getRootItem() { return nullptr; }
+
 protected:
     QString _sourceUrl;
     uint16_t _dpi;

scripts/defaultScripts.js

@@ -34,7 +34,8 @@ var DEFAULT_SCRIPTS = [
     "system/firstPersonHMD.js",
     "system/snapshot.js",
     "system/help.js",
-    "system/bubble.js"
+    "system/bubble.js",
+    "system/tablet-ui/tabletUI.js"
 ];
 
 // add a menu item for debugging

scripts/system/controllers/handControllerGrab.js

@@ -1267,6 +1267,9 @@ function MyController(hand) {
         var grabbableProps = entityPropertiesCache.getGrabbableProps(entityID);
         var grabProps = entityPropertiesCache.getGrabProps(entityID);
         var props = entityPropertiesCache.getProps(entityID);
+        if (!props) {
+            return false;
+        }
         var physical = propsArePhysical(props);
         var grabbable = false;
         var debug = (WANT_DEBUG_SEARCH_NAME && props.name === WANT_DEBUG_SEARCH_NAME);
@@ -1546,7 +1549,7 @@ function MyController(hand) {
                 Entities.sendHoverOverEntity(entity, pointerEvent);
             }
 
-            if (this.triggerSmoothedGrab() && !isEditing()) {
+            if (this.triggerSmoothedGrab() && (!isEditing() || name == "WebTablet Web")) {
                 this.grabbedEntity = entity;
                 this.setState(STATE_ENTITY_TOUCHING, "begin touching entity '" + name + "'");
                 return;

scripts/system/libraries/WebTablet.js

@@ -39,54 +39,72 @@ function calcSpawnInfo() {
 }
 
 // ctor
-WebTablet = function (url, width, dpi, clientOnly) {
+WebTablet = function (url, width, dpi, location, clientOnly) {
 
     var ASPECT = 4.0 / 3.0;
     var WIDTH = width || DEFAULT_WIDTH;
     var HEIGHT = WIDTH * ASPECT;
     var DEPTH = 0.025;
     var DPI = dpi || DEFAULT_DPI;
+    var _this = this;
 
-    var spawnInfo = calcSpawnInfo();
+    var tabletProperties = {
-
+        name: "WebTablet Tablet",
-    var tabletEntityPosition = spawnInfo.position;
-    var tabletEntityRotation = spawnInfo.rotation;
-    this.tabletEntityID = Entities.addEntity({
-        name: "tablet",
         type: "Model",
         modelURL: TABLET_URL,
-        position: tabletEntityPosition,
-        rotation: tabletEntityRotation,
         userData: JSON.stringify({
             "grabbableKey": {"grabbable": true}
         }),
         dimensions: {x: WIDTH, y: HEIGHT, z: DEPTH},
         parentID: MyAvatar.sessionUUID,
         parentJointIndex: -2
-    }, clientOnly);
+    }
+
+    if (location) {
+        tabletProperties.localPosition = location.localPosition;
+        tabletProperties.localRotation = location.localRotation;
+    } else {
+        var spawnInfo = calcSpawnInfo();
+        tabletProperties.position = spawnInfo.position;
+        tabletProperties.rotation = spawnInfo.rotation;
+    }
+
+    this.tabletEntityID = Entities.addEntity(tabletProperties, clientOnly);
 
     var WEB_ENTITY_REDUCTION_FACTOR = {x: 0.78, y: 0.85};
     var WEB_ENTITY_Z_OFFSET = -0.01;
 
-    var webEntityRotation = Quat.multiply(spawnInfo.rotation, Quat.angleAxis(180, Y_AXIS));
+    this.createWebEntity = function(url) {
-    var webEntityPosition = Vec3.sum(spawnInfo.position, Vec3.multiply(WEB_ENTITY_Z_OFFSET, Quat.getFront(webEntityRotation)));
+        if (_this.webEntityID) {
+            Entities.deleteEntity(_this.webEntityID);
+        }
+        _this.webEntityID = Entities.addEntity({
+            name: "WebTablet Web",
+            type: "Web",
+            sourceUrl: url,
+            dimensions: {x: WIDTH * WEB_ENTITY_REDUCTION_FACTOR.x,
+                         y: HEIGHT * WEB_ENTITY_REDUCTION_FACTOR.y,
+                         z: 0.1},
+            localPosition: { x: 0, y: 0, z: WEB_ENTITY_Z_OFFSET },
+            localRotation: Quat.angleAxis(180, Y_AXIS),
+            shapeType: "box",
+            dpi: DPI,
+            parentID: _this.tabletEntityID,
+            parentJointIndex: -1
+        }, clientOnly);
+    }
 
-    this.webEntityID = Entities.addEntity({
+    this.createWebEntity(url);
-        name: "web",
-        type: "Web",
-        sourceUrl: url,
-        dimensions: {x: WIDTH * WEB_ENTITY_REDUCTION_FACTOR.x,
-                     y: HEIGHT * WEB_ENTITY_REDUCTION_FACTOR.y,
-                     z: 0.1},
-        position: webEntityPosition,
-        rotation: webEntityRotation,
-        shapeType: "box",
-        dpi: DPI,
-        parentID: this.tabletEntityID,
-        parentJointIndex: -1
-    }, clientOnly);
 
     this.state = "idle";
+
+    this.getRoot = function() {
+        return Entities.getWebViewRoot(_this.webEntityID);
+    }
+
+    this.getLocation = function() {
+        return Entities.getEntityProperties(_this.tabletEntityID, ["localPosition", "localRotation"]);
+    };
 };
 
 WebTablet.prototype.destroy = function () {

scripts/system/libraries/entitySelectionTool.js

@@ -1032,9 +1032,16 @@ SelectionDisplay = (function() {
                     var pickRay = controllerComputePickRay();
                     if (pickRay) {
                         var entityIntersection = Entities.findRayIntersection(pickRay, true);
+
+
                         var overlayIntersection = Overlays.findRayIntersection(pickRay);
                         if (entityIntersection.intersects &&
                             (!overlayIntersection.intersects || (entityIntersection.distance < overlayIntersection.distance))) {
+
+                            if (HMD.tabletID == entityIntersection.entityID) {
+                                return;
+                            }
+
                             selectionManager.setSelections([entityIntersection.entityID]);
                         }
                     }

scripts/system/marketplaces/marketplace.js

@@ -44,7 +44,17 @@ function shouldShowWebTablet() {
 function showMarketplace(marketplaceID) {
     if (shouldShowWebTablet()) {
         updateButtonState(true);
-        marketplaceWebTablet = new WebTablet("https://metaverse.highfidelity.com/marketplace", null, null, true);
+
+        if (HMD.tabletID) {
+            UIWebTablet.createWebEntity("https://metaverse.highfidelity.com/marketplace");
+            HMD.tabletID = UIWebTablet.webEntityID;
+        } else {
+            marketplaceWebTablet = new WebTablet("https://metaverse.highfidelity.com/marketplace",
+                                                 null, // width
+                                                 null, // dpi
+                                                 null, // location
+                                                 true); // client-only
+        }
         Settings.setValue(persistenceKey, marketplaceWebTablet.pickle());
     } else {
         var url = MARKETPLACE_URL;

scripts/system/tablet-ui/tabletUI.js

@@ -0,0 +1,68 @@
+"use strict";
+
+//
+//  tabletUI.js
+//
+//  scripts/system/tablet-ui/
+//
+//  Created by Seth Alves 2016-9-29
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+/* global Script, HMD, WebTablet, UIWebTablet */
+
+(function() { // BEGIN LOCAL_SCOPE
+    var tabletShown = false;
+    var tabletLocation = null;
+
+    Script.include("../libraries/WebTablet.js");
+
+    function showTabletUI() {
+        tabletShown = true;
+        print("show tablet-ui");
+        UIWebTablet = new WebTablet("qml/hifi/tablet/Tablet.qml", null, null, tabletLocation);
+        HMD.tabletID = UIWebTablet.webEntityID;
+
+        var setUpTabletUI = function() {
+            var root = UIWebTablet.getRoot();
+            if (!root) {
+                print("HERE no root yet");
+                Script.setTimeout(setUpTabletUI, 100);
+                return;
+            }
+            print("HERE got root", root);
+        }
+
+        Script.setTimeout(setUpTabletUI, 100);
+    }
+
+    function hideTabletUI() {
+        tabletShown = false;
+        print("hide tablet-ui");
+        if (UIWebTablet) {
+            if (UIWebTablet.onClose) {
+                UIWebTablet.onClose();
+            }
+
+            tabletLocation = UIWebTablet.getLocation();
+            UIWebTablet.destroy();
+            UIWebTablet = null;
+            HMD.tabletID = null;
+        }
+    }
+
+    function updateShowTablet() {
+        if (HMD.showTablet && !tabletShown) {
+            showTabletUI();
+        } else if (!HMD.showTablet && tabletShown) {
+            hideTabletUI();
+        }
+    }
+
+    Script.update.connect(updateShowTablet);
+    // Script.setInterval(updateShowTablet, 1000);
+
+}()); // END LOCAL_SCOPE