Merge pull request #6959 from Triplelexx/20769

CR for Job #20769 - Touchscreen camera input for Interface

interface/resources/controllers/touchscreen.json

@@ -0,0 +1,23 @@
+{
+    "name": "Touchscreen to Actions",
+    "channels": [
+        { "from": "Touchscreen.GesturePinchOut", "to": "Actions.BoomOut", "filters": [ { "type": "scale", "scale": 0.02 } ] },
+        { "from": "Touchscreen.GesturePinchIn", "to": "Actions.BoomIn", "filters": [ { "type": "scale", "scale": 0.02 } ] },
+
+        { "from": { "makeAxis" : [
+              [ "Touchscreen.DragLeft" ],
+              [ "Touchscreen.DragRight" ] 
+          ] 
+        },
+          "to": "Actions.Yaw", "filters": [ { "type": "scale", "scale": 0.12 } ]
+        },
+
+        { "from": { "makeAxis" : [
+                [ "Touchscreen.DragUp" ],
+                [ "Touchscreen.DragDown" ] 
+            ] 
+          },
+          "to": "Actions.Pitch", "filters": [ { "type": "scale", "scale": 0.04 } ]
+        }	
+    ]
+}

interface/src/Application.cpp

@@ -957,8 +957,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) :
         return DependencyManager::get<OffscreenUi>()->navigationFocused() ? 1 : 0;
     });
 
-    // Setup the keyboardMouseDevice and the user input mapper with the default bindings
+    // Setup the _keyboardMouseDevice, _touchscreenDevice and the user input mapper with the default bindings
     userInputMapper->registerDevice(_keyboardMouseDevice->getInputDevice());
+    // if the _touchscreenDevice is not supported it will not be registered
+    if (_touchscreenDevice) {
+        userInputMapper->registerDevice(_touchscreenDevice->getInputDevice());
+    }
 
     // force the model the look at the correct directory (weird order of operations issue)
     scriptEngines->setScriptsLocation(scriptEngines->getScriptsLocation());
@@ -1600,6 +1604,9 @@ void Application::initializeUi() {
         if (KeyboardMouseDevice::NAME == inputPlugin->getName()) {
             _keyboardMouseDevice = std::dynamic_pointer_cast<KeyboardMouseDevice>(inputPlugin);
         }
+        if (TouchscreenDevice::NAME == inputPlugin->getName()) {
+            _touchscreenDevice = std::dynamic_pointer_cast<TouchscreenDevice>(inputPlugin);
+        }
     }
     _window->setMenuBar(new Menu());
 
@@ -2096,6 +2103,9 @@ bool Application::event(QEvent* event) {
         case QEvent::TouchUpdate:
             touchUpdateEvent(static_cast<QTouchEvent*>(event));
             return true;
+        case QEvent::Gesture:
+            touchGestureEvent((QGestureEvent*)event);
+            return true;
         case QEvent::Wheel:
             wheelEvent(static_cast<QWheelEvent*>(event));
             return true;
@@ -2727,6 +2737,9 @@ void Application::touchUpdateEvent(QTouchEvent* event) {
     if (_keyboardMouseDevice->isActive()) {
         _keyboardMouseDevice->touchUpdateEvent(event);
     }
+    if (_touchscreenDevice->isActive()) {
+        _touchscreenDevice->touchUpdateEvent(event);
+    }
 }
 
 void Application::touchBeginEvent(QTouchEvent* event) {
@@ -2745,6 +2758,9 @@ void Application::touchBeginEvent(QTouchEvent* event) {
     if (_keyboardMouseDevice->isActive()) {
         _keyboardMouseDevice->touchBeginEvent(event);
     }
+    if (_touchscreenDevice->isActive()) {
+        _touchscreenDevice->touchBeginEvent(event);
+    }
 
 }
 
@@ -2762,10 +2778,19 @@ void Application::touchEndEvent(QTouchEvent* event) {
     if (_keyboardMouseDevice->isActive()) {
         _keyboardMouseDevice->touchEndEvent(event);
     }
+    if (_touchscreenDevice->isActive()) {
+        _touchscreenDevice->touchEndEvent(event);
+    }
 
     // put any application specific touch behavior below here..
 }
 
+void Application::touchGestureEvent(QGestureEvent* event) {
+    if (_touchscreenDevice->isActive()) {
+        _touchscreenDevice->touchGestureEvent(event);
+    }
+}
+
 void Application::wheelEvent(QWheelEvent* event) const {
     _altPressed = false;
     _controllerScriptingInterface->emitWheelEvent(event); // send events to any registered scripts

interface/src/Application.h

@@ -29,6 +29,7 @@
 #include <EntityEditPacketSender.h>
 #include <EntityTreeRenderer.h>
 #include <input-plugins/KeyboardMouseDevice.h>
+#include <input-plugins/TouchscreenDevice.h>
 #include <OctreeQuery.h>
 #include <PhysicalEntitySimulation.h>
 #include <PhysicsEngine.h>
@@ -403,6 +404,7 @@ private:
     void touchBeginEvent(QTouchEvent* event);
     void touchEndEvent(QTouchEvent* event);
     void touchUpdateEvent(QTouchEvent* event);
+    void touchGestureEvent(QGestureEvent* event);
 
     void wheelEvent(QWheelEvent* event) const;
     void dropEvent(QDropEvent* event);
@@ -455,6 +457,7 @@ private:
 
     std::shared_ptr<controller::StateController> _applicationStateDevice; // Default ApplicationDevice reflecting the state of different properties of the session
     std::shared_ptr<KeyboardMouseDevice> _keyboardMouseDevice;   // Default input device, the good old keyboard mouse and maybe touchpad
+    std::shared_ptr<TouchscreenDevice> _touchscreenDevice;   // the good old touchscreen
     SimpleMovingAverage _avatarSimsPerSecond {10};
     int _avatarSimsPerSecondReport {0};
     quint64 _lastAvatarSimsPerSecondUpdate {0};

libraries/gl/src/gl/GLWidget.cpp

@@ -43,6 +43,7 @@ int GLWidget::getDeviceHeight() const {
 
 void GLWidget::initializeGL() {
     setAttribute(Qt::WA_AcceptTouchEvents);
+    grabGesture(Qt::PinchGesture);
     setAcceptDrops(true);
     // Note, we *DO NOT* want Qt to automatically swap buffers for us.  This results in the "ringing" bug mentioned in WL#19514 when we're throttling the framerate.
     setAutoBufferSwap(false);
@@ -81,6 +82,7 @@ bool GLWidget::event(QEvent* event) {
         case QEvent::TouchBegin:
         case QEvent::TouchEnd:
         case QEvent::TouchUpdate:
+        case QEvent::Gesture:
         case QEvent::Wheel:
         case QEvent::DragEnter:
         case QEvent::Drop:

libraries/input-plugins/src/input-plugins/InputPlugin.cpp

@@ -13,11 +13,13 @@
 #include <plugins/PluginManager.h>
 
 #include "KeyboardMouseDevice.h"
+#include "TouchscreenDevice.h"
 
 // TODO migrate to a DLL model where plugins are discovered and loaded at runtime by the PluginManager class
 InputPluginList getInputPlugins() {
     InputPlugin* PLUGIN_POOL[] = {
         new KeyboardMouseDevice(),
+        new TouchscreenDevice(),
         nullptr
     };
 

libraries/input-plugins/src/input-plugins/KeyboardMouseDevice.cpp

@@ -19,6 +19,7 @@
 #include <NumericalConstants.h>
 
 const QString KeyboardMouseDevice::NAME = "Keyboard/Mouse";
+bool KeyboardMouseDevice::_enableTouch = true;
 
 void KeyboardMouseDevice::pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
     auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
@@ -79,7 +80,7 @@ void KeyboardMouseDevice::mouseReleaseEvent(QMouseEvent* event) {
 
     // if we pressed and released at the same location within a small time window, then create a "_CLICKED" 
     // input for this button we might want to add some small tolerance to this so if you do a small drag it 
-    // till counts as a clicked.
+    // still counts as a click.
     static const int CLICK_TIME = USECS_PER_MSEC * 500; // 500 ms to click
     if (!_mouseMoved && (usecTimestampNow() - _mousePressTime < CLICK_TIME)) {
         _inputDevice->_buttonPressedMap.insert(_inputDevice->makeInput((Qt::MouseButton) event->button(), true).getChannel());
@@ -132,18 +133,23 @@ glm::vec2 evalAverageTouchPoints(const QList<QTouchEvent::TouchPoint>& points) {
 }
 
 void KeyboardMouseDevice::touchBeginEvent(const QTouchEvent* event) {
+    if (_enableTouch) {
         _isTouching = event->touchPointStates().testFlag(Qt::TouchPointPressed);
         _lastTouch = evalAverageTouchPoints(event->touchPoints());
         _lastTouchTime = _clock.now();
+    }
 }
 
 void KeyboardMouseDevice::touchEndEvent(const QTouchEvent* event) {
+    if (_enableTouch) {
         _isTouching = false;
         _lastTouch = evalAverageTouchPoints(event->touchPoints());
         _lastTouchTime = _clock.now();
+    }
 }
 
 void KeyboardMouseDevice::touchUpdateEvent(const QTouchEvent* event) {
+    if (_enableTouch) {
         auto currentPos = evalAverageTouchPoints(event->touchPoints());
         _lastTouchTime = _clock.now();
 
@@ -160,6 +166,7 @@ void KeyboardMouseDevice::touchUpdateEvent(const QTouchEvent* event) {
         }
 
         _lastTouch = currentPos;
+    }
 }
 
 controller::Input KeyboardMouseDevice::InputDevice::makeInput(Qt::Key code) const {
@@ -247,4 +254,3 @@ QString KeyboardMouseDevice::InputDevice::getDefaultMappingConfig() const {
     static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/keyboardMouse.json";
     return MAPPING_JSON;
 }
-

libraries/input-plugins/src/input-plugins/KeyboardMouseDevice.h

@@ -85,6 +85,8 @@ public:
 
     void wheelEvent(QWheelEvent* event);
 
+    static void enableTouch(bool enableTouch) { _enableTouch = enableTouch; }
+    
     static const QString NAME;
 
 protected:
@@ -122,6 +124,8 @@ protected:
     bool _isTouching = false;
     std::chrono::high_resolution_clock _clock;
     std::chrono::high_resolution_clock::time_point _lastTouchTime;
+
+    static bool _enableTouch;
 };
 
 #endif // hifi_KeyboardMouseDevice_h

libraries/input-plugins/src/input-plugins/TouchscreenDevice.cpp

@@ -0,0 +1,132 @@
+//
+//  TouchscreenDevice.cpp
+//  input-plugins/src/input-plugins
+//
+//  Created by Triplelexx on 01/31/16.
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "TouchscreenDevice.h"
+#include "KeyboardMouseDevice.h"
+
+#include <QtGui/QTouchEvent>
+#include <QGestureEvent>
+#include <QGuiApplication>
+#include <QWindow>
+#include <QScreen>
+
+#include <controllers/UserInputMapper.h>
+#include <PathUtils.h>
+#include <NumericalConstants.h>
+
+const QString TouchscreenDevice::NAME = "Touchscreen";
+
+bool TouchscreenDevice::isSupported() const {
+    for (auto touchDevice : QTouchDevice::devices()) {
+        if (touchDevice->type() == QTouchDevice::TouchScreen) {
+            return true;
+        }
+    }
+    return false;
+}
+
+void TouchscreenDevice::pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
+    auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
+    userInputMapper->withLock([&, this]() {
+        _inputDevice->update(deltaTime, inputCalibrationData);
+    });
+
+    float distanceScaleX, distanceScaleY;
+    if (_touchPointCount == 1) {
+        if (_firstTouchVec.x < _currentTouchVec.x) {
+            distanceScaleX = (_currentTouchVec.x - _firstTouchVec.x) / _screenDPIScale.x;
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_AXIS_X_POS).getChannel()] = distanceScaleX;
+        } else if (_firstTouchVec.x > _currentTouchVec.x) {
+            distanceScaleX = (_firstTouchVec.x - _currentTouchVec.x) / _screenDPIScale.x;
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_AXIS_X_NEG).getChannel()] = distanceScaleX;
+        }
+        // Y axis is inverted, positive is pointing up the screen
+        if (_firstTouchVec.y > _currentTouchVec.y) {
+            distanceScaleY = (_firstTouchVec.y - _currentTouchVec.y) / _screenDPIScale.y;
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_AXIS_Y_POS).getChannel()] = distanceScaleY;
+        } else if (_firstTouchVec.y < _currentTouchVec.y) {
+            distanceScaleY = (_currentTouchVec.y - _firstTouchVec.y) / _screenDPIScale.y;
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_AXIS_Y_NEG).getChannel()] = distanceScaleY;
+        }
+    } else  if (_touchPointCount == 2) {
+        if (_pinchScale > _lastPinchScale && _pinchScale != 0) {
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_GESTURE_PINCH_POS).getChannel()] = 1.0f;
+        } else if (_pinchScale != 0) {
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_GESTURE_PINCH_NEG).getChannel()] = 1.0f;
+        }
+        _lastPinchScale = _pinchScale;
+    }
+}
+
+void TouchscreenDevice::InputDevice::update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
+    _axisStateMap.clear();
+}
+
+void TouchscreenDevice::InputDevice::focusOutEvent() {
+}
+
+void TouchscreenDevice::touchBeginEvent(const QTouchEvent* event) {
+    const QTouchEvent::TouchPoint& point = event->touchPoints().at(0);
+    _firstTouchVec = glm::vec2(point.pos().x(), point.pos().y());
+    KeyboardMouseDevice::enableTouch(false);
+    QScreen* eventScreen = event->window()->screen();
+    if (_screenDPI != eventScreen->physicalDotsPerInch()) {
+        _screenDPIScale.x = (float)eventScreen->physicalDotsPerInchX();
+        _screenDPIScale.y = (float)eventScreen->physicalDotsPerInchY();
+        _screenDPI = eventScreen->physicalDotsPerInch();
+    }
+}
+
+void TouchscreenDevice::touchEndEvent(const QTouchEvent* event) {
+    _touchPointCount = 0;
+    KeyboardMouseDevice::enableTouch(true);
+}
+
+void TouchscreenDevice::touchUpdateEvent(const QTouchEvent* event) {
+    const QTouchEvent::TouchPoint& point = event->touchPoints().at(0);
+    _currentTouchVec = glm::vec2(point.pos().x(), point.pos().y());
+    _touchPointCount = event->touchPoints().count();
+}
+
+void TouchscreenDevice::touchGestureEvent(const QGestureEvent* event) {
+    if (QGesture* gesture = event->gesture(Qt::PinchGesture)) {
+        QPinchGesture* pinch = static_cast<QPinchGesture*>(gesture);
+        _pinchScale = pinch->totalScaleFactor();
+    }
+}
+
+controller::Input TouchscreenDevice::InputDevice::makeInput(TouchscreenDevice::TouchAxisChannel axis) const {
+    return controller::Input(_deviceID, axis, controller::ChannelType::AXIS);
+}
+
+controller::Input TouchscreenDevice::InputDevice::makeInput(TouchscreenDevice::TouchGestureAxisChannel gesture) const {
+    return controller::Input(_deviceID, gesture, controller::ChannelType::AXIS);
+}
+
+controller::Input::NamedVector TouchscreenDevice::InputDevice::getAvailableInputs() const {
+    using namespace controller;
+    static QVector<Input::NamedPair> availableInputs;
+    static std::once_flag once;
+    std::call_once(once, [&] {
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_AXIS_X_POS), "DragRight"));
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_AXIS_X_NEG), "DragLeft"));
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_AXIS_Y_POS), "DragUp"));
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_AXIS_Y_NEG), "DragDown"));
+
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_GESTURE_PINCH_POS), "GesturePinchOut"));
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_GESTURE_PINCH_NEG), "GesturePinchIn"));
+    });
+    return availableInputs;
+}
+
+QString TouchscreenDevice::InputDevice::getDefaultMappingConfig() const {
+    static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/touchscreen.json";
+    return MAPPING_JSON;
+}

libraries/input-plugins/src/input-plugins/TouchscreenDevice.h

@@ -0,0 +1,84 @@
+//
+//  TouchscreenDevice.h
+//  input-plugins/src/input-plugins
+//
+//  Created by Triplelexx on 1/31/16.
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_TouchscreenDevice_h
+#define hifi_TouchscreenDevice_h
+
+#include <controllers/InputDevice.h>
+#include "InputPlugin.h"
+#include <QtGui/qtouchdevice.h>
+
+class QTouchEvent;
+class QGestureEvent;
+
+class TouchscreenDevice : public InputPlugin {
+    Q_OBJECT
+public:
+        
+    enum TouchAxisChannel {
+        TOUCH_AXIS_X_POS = 0,
+        TOUCH_AXIS_X_NEG,
+        TOUCH_AXIS_Y_POS,
+        TOUCH_AXIS_Y_NEG,
+    };
+
+	enum TouchGestureAxisChannel {
+        TOUCH_GESTURE_PINCH_POS = TOUCH_AXIS_Y_NEG + 1,
+        TOUCH_GESTURE_PINCH_NEG,
+    };
+
+    // Plugin functions
+    virtual bool isSupported() const override;
+    virtual const QString& getName() const override { return NAME; }
+
+    virtual void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); }
+    virtual void pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override;
+
+    void touchBeginEvent(const QTouchEvent* event);
+    void touchEndEvent(const QTouchEvent* event);
+    void touchUpdateEvent(const QTouchEvent* event);
+    void touchGestureEvent(const QGestureEvent* event);
+    
+    static const QString NAME;
+
+protected:
+
+    class InputDevice : public controller::InputDevice {
+    public:
+        InputDevice() : controller::InputDevice("Touchscreen") {}
+    private:
+        // Device functions
+        virtual controller::Input::NamedVector getAvailableInputs() const override;
+        virtual QString getDefaultMappingConfig() const override;
+        virtual void update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override;
+        virtual void focusOutEvent() override;
+
+        controller::Input makeInput(TouchAxisChannel axis) const;
+        controller::Input makeInput(TouchGestureAxisChannel gesture) const;
+
+        friend class TouchscreenDevice;
+    };
+
+public:
+    const std::shared_ptr<InputDevice>& getInputDevice() const { return _inputDevice; }
+
+protected:
+    qreal _lastPinchScale;
+    qreal _pinchScale;
+    qreal _screenDPI;
+    glm::vec2 _screenDPIScale;
+    glm::vec2 _firstTouchVec;
+    glm::vec2 _currentTouchVec;
+    int _touchPointCount;
+    std::shared_ptr<InputDevice> _inputDevice { std::make_shared<InputDevice>() };
+};
+
+#endif // hifi_TouchscreenDevice_h

tests/controllers/src/main.cpp

@@ -38,6 +38,7 @@
 #include <plugins/PluginManager.h>
 #include <input-plugins/InputPlugin.h>
 #include <input-plugins/KeyboardMouseDevice.h>
+#include <input-plugins/TouchscreenDevice.h>
 #include <controllers/ScriptingInterface.h>
 
 #include <DependencyManager.h>
@@ -144,6 +145,9 @@ int main(int argc, char** argv) {
             if (name == KeyboardMouseDevice::NAME) {
                 userInputMapper->registerDevice(std::dynamic_pointer_cast<KeyboardMouseDevice>(inputPlugin)->getInputDevice());
             }
+            if (name == TouchscreenDevice::NAME) {
+                userInputMapper->registerDevice(std::dynamic_pointer_cast<TouchscreenDevice>(inputPlugin)->getInputDevice());
+            }
             inputPlugin->pluginUpdate(0, calibrationData);
         }
         rootContext->setContextProperty("Controllers", new MyControllerScriptingInterface());