revise touchscreen camera control manipulation

Touchscreen camera control is now via a touchscreen device. Input must
be enabled with the menu option. Currently supports dragging and
gesturing to control avatar camera. Gesturing is handled by integration
of the Qt implementation.

interface/resources/controllers/touchscreen.json

@@ -0,0 +1,24 @@
+{
+    "name": "Touchscreen to Actions",
+    "channels": [
+
+        { "from": "Touchscreen.GesturePinchOut", "to": "Actions.BoomOut", "filters": [ { "type": "scale", "scale": 0.02 } ]},
+        { "from": "Touchscreen.GesturePinchIn", "to": "Actions.BoomIn", "filters": [ { "type": "scale", "scale": 0.02 } ]},
+		
+        { "from": { "makeAxis" : [
+                [ "Touchscreen.DragLeft" ],
+                [ "Touchscreen.DragRight" ] 
+            ] 
+          },
+          "to": "Actions.Yaw"
+        },
+		
+		{ "from": { "makeAxis" : [
+                [ "Touchscreen.DragUp" ],
+                [ "Touchscreen.DragDown" ] 
+            ] 
+          },
+          "to": "Actions.Pitch"
+        }	
+    ]
+}

interface/src/Application.cpp

@@ -862,8 +862,9 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) :
 
     userInputMapper->registerDevice(_applicationStateDevice);
 
-    // Setup the keyboardMouseDevice and the user input mapper with the default bindings
+    // Setup the _keyboardMouseDevice, _touchscreenDevice and the user input mapper with the default bindings
     userInputMapper->registerDevice(_keyboardMouseDevice->getInputDevice());
+    userInputMapper->registerDevice(_touchscreenDevice->getInputDevice());
     userInputMapper->loadDefaultMapping(userInputMapper->getStandardDeviceID());
 
     // force the model the look at the correct directory (weird order of operations issue)
@@ -1308,6 +1309,9 @@ void Application::initializeUi() {
         if (name == KeyboardMouseDevice::NAME) {
             _keyboardMouseDevice = std::dynamic_pointer_cast<KeyboardMouseDevice>(inputPlugin);
         }
+        if (name == TouchscreenDevice::NAME) {
+            _touchscreenDevice = std::dynamic_pointer_cast<TouchscreenDevice>(inputPlugin);
+        }
     }
     updateInputModes();
 }
@@ -1790,6 +1794,9 @@ bool Application::event(QEvent* event) {
         case QEvent::TouchUpdate:
             touchUpdateEvent(static_cast<QTouchEvent*>(event));
             return true;
+        case QEvent::Gesture:
+            touchGestureEvent((QGestureEvent*)event);
+            return true;
         case QEvent::Wheel:
             wheelEvent(static_cast<QWheelEvent*>(event));
             return true;
@@ -2336,6 +2343,9 @@ void Application::touchUpdateEvent(QTouchEvent* event) {
     if (Menu::getInstance()->isOptionChecked(KeyboardMouseDevice::NAME)) {
         _keyboardMouseDevice->touchUpdateEvent(event);
     }
+    if (Menu::getInstance()->isOptionChecked(TouchscreenDevice::NAME)) {
+        _touchscreenDevice->touchUpdateEvent(event);
+    }
 }
 
 void Application::touchBeginEvent(QTouchEvent* event) {
@@ -2354,6 +2364,9 @@ void Application::touchBeginEvent(QTouchEvent* event) {
     if (Menu::getInstance()->isOptionChecked(KeyboardMouseDevice::NAME)) {
         _keyboardMouseDevice->touchBeginEvent(event);
     }
+    if (Menu::getInstance()->isOptionChecked(TouchscreenDevice::NAME)) {
+        _touchscreenDevice->touchBeginEvent(event);
+    }
 
 }
 
@@ -2371,10 +2384,19 @@ void Application::touchEndEvent(QTouchEvent* event) {
     if (Menu::getInstance()->isOptionChecked(KeyboardMouseDevice::NAME)) {
         _keyboardMouseDevice->touchEndEvent(event);
     }
+    if (Menu::getInstance()->isOptionChecked(TouchscreenDevice::NAME)) {
+        _touchscreenDevice->touchEndEvent(event);
+    }
 
     // put any application specific touch behavior below here..
 }
 
+void Application::touchGestureEvent(QGestureEvent* event) {
+    if (Menu::getInstance()->isOptionChecked(TouchscreenDevice::NAME)) {
+        _touchscreenDevice->touchGestureEvent(event);
+    }
+}
+
 void Application::wheelEvent(QWheelEvent* event) {
     _altPressed = false;
     _controllerScriptingInterface->emitWheelEvent(event); // send events to any registered scripts

interface/src/Application.h

@@ -29,6 +29,7 @@
 #include <EntityEditPacketSender.h>
 #include <EntityTreeRenderer.h>
 #include <input-plugins/KeyboardMouseDevice.h>
+#include <input-plugins/TouchscreenDevice.h>
 #include <OctreeQuery.h>
 #include <PhysicalEntitySimulation.h>
 #include <PhysicsEngine.h>
@@ -376,6 +377,7 @@ private:
     void touchBeginEvent(QTouchEvent* event);
     void touchEndEvent(QTouchEvent* event);
     void touchUpdateEvent(QTouchEvent* event);
+    void touchGestureEvent(QGestureEvent* event);
 
     void wheelEvent(QWheelEvent* event);
     void dropEvent(QDropEvent* event);
@@ -421,6 +423,7 @@ private:
 
     std::shared_ptr<controller::StateController> _applicationStateDevice; // Default ApplicationDevice reflecting the state of different properties of the session
     std::shared_ptr<KeyboardMouseDevice> _keyboardMouseDevice;   // Default input device, the good old keyboard mouse and maybe touchpad
+    std::shared_ptr<TouchscreenDevice> _touchscreenDevice;   // the good old touchscreen
     AvatarUpdate* _avatarUpdate {nullptr};
     SimpleMovingAverage _avatarSimsPerSecond {10};
     int _avatarSimsPerSecondReport {0};

libraries/gl/src/gl/GLWidget.cpp

@@ -43,6 +43,7 @@ int GLWidget::getDeviceHeight() const {
 
 void GLWidget::initializeGL() {
     setAttribute(Qt::WA_AcceptTouchEvents);
+    grabGesture(Qt::PinchGesture);
     setAcceptDrops(true);
     // Note, we *DO NOT* want Qt to automatically swap buffers for us.  This results in the "ringing" bug mentioned in WL#19514 when we're throttling the framerate.
     setAutoBufferSwap(false);
@@ -76,6 +77,7 @@ bool GLWidget::event(QEvent* event) {
         case QEvent::TouchBegin:
         case QEvent::TouchEnd:
         case QEvent::TouchUpdate:
+        case QEvent::Gesture:
         case QEvent::Wheel:
         case QEvent::DragEnter:
         case QEvent::Drop:

libraries/input-plugins/src/input-plugins/InputPlugin.cpp

@@ -13,11 +13,13 @@
 #include <plugins/PluginManager.h>
 
 #include "KeyboardMouseDevice.h"
+#include "TouchscreenDevice.h"
 
 // TODO migrate to a DLL model where plugins are discovered and loaded at runtime by the PluginManager class
 InputPluginList getInputPlugins() {
     InputPlugin* PLUGIN_POOL[] = {
         new KeyboardMouseDevice(),
+        new TouchscreenDevice(),
         nullptr
     };
 

libraries/input-plugins/src/input-plugins/KeyboardMouseDevice.cpp

@@ -19,6 +19,7 @@
 #include <NumericalConstants.h>
 
 const QString KeyboardMouseDevice::NAME = "Keyboard/Mouse";
+bool KeyboardMouseDevice::_enableMouse = true;
 
 void KeyboardMouseDevice::pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData, bool jointsCaptured) { 
     _inputDevice->update(deltaTime, inputCalibrationData, jointsCaptured); 
@@ -58,28 +59,32 @@ void KeyboardMouseDevice::keyReleaseEvent(QKeyEvent* event) {
 }
 
 void KeyboardMouseDevice::mousePressEvent(QMouseEvent* event, unsigned int deviceID) {
-    auto input = _inputDevice->makeInput((Qt::MouseButton) event->button());
-    auto result = _inputDevice->_buttonPressedMap.insert(input.getChannel());
-    if (!result.second) {
-        // key pressed again ? without catching the release event ?
-    }
-    _lastCursor = event->pos();
-    _mousePressTime = usecTimestampNow();
-    _mouseMoved = false;
+    if (_enableMouse) {
+        auto input = _inputDevice->makeInput((Qt::MouseButton) event->button());
+        auto result = _inputDevice->_buttonPressedMap.insert(input.getChannel());
+        if (!result.second) {
+            // key pressed again ? without catching the release event ?
+        }
+        _lastCursor = event->pos();
+        _mousePressTime = usecTimestampNow();
+        _mouseMoved = false;
 
-    eraseMouseClicked();
+        eraseMouseClicked();
+    }
 }
 
 void KeyboardMouseDevice::mouseReleaseEvent(QMouseEvent* event, unsigned int deviceID) {
-    auto input = _inputDevice->makeInput((Qt::MouseButton) event->button());
-    _inputDevice->_buttonPressedMap.erase(input.getChannel());
+    if (_enableMouse) {
+        auto input = _inputDevice->makeInput((Qt::MouseButton) event->button());
+        _inputDevice->_buttonPressedMap.erase(input.getChannel());
 
-    // if we pressed and released at the same location within a small time window, then create a "_CLICKED" 
-    // input for this button we might want to add some small tolerance to this so if you do a small drag it 
-    // till counts as a clicked.
-    static const int CLICK_TIME = USECS_PER_MSEC * 500; // 500 ms to click
-    if (!_mouseMoved && (usecTimestampNow() - _mousePressTime < CLICK_TIME)) {
-        _inputDevice->_buttonPressedMap.insert(_inputDevice->makeInput((Qt::MouseButton) event->button(), true).getChannel());
+        // if we pressed and released at the same location within a small time window, then create a "_CLICKED" 
+        // input for this button we might want to add some small tolerance to this so if you do a small drag it 
+        // still counts as a click.
+        static const int CLICK_TIME = USECS_PER_MSEC * 500; // 500 ms to click
+        if (!_mouseMoved && (usecTimestampNow() - _mousePressTime < CLICK_TIME)) {
+            _inputDevice->_buttonPressedMap.insert(_inputDevice->makeInput((Qt::MouseButton) event->button(), true).getChannel());
+        }
     }
 }
 
@@ -90,22 +95,24 @@ void KeyboardMouseDevice::eraseMouseClicked() {
 }
 
 void KeyboardMouseDevice::mouseMoveEvent(QMouseEvent* event, unsigned int deviceID) {
-    QPoint currentPos = event->pos();
-    QPoint currentMove = currentPos - _lastCursor;
+    if (_enableMouse) {
+        QPoint currentPos = event->pos();
+        QPoint currentMove = currentPos - _lastCursor;
 
-    _inputDevice->_axisStateMap[MOUSE_AXIS_X_POS] = (currentMove.x() > 0 ? currentMove.x() : 0.0f);
-    _inputDevice->_axisStateMap[MOUSE_AXIS_X_NEG] = (currentMove.x() < 0 ? -currentMove.x() : 0.0f);
-     // Y mouse is inverted positive is pointing up the screen
-    _inputDevice->_axisStateMap[MOUSE_AXIS_Y_POS] = (currentMove.y() < 0 ? -currentMove.y() : 0.0f);
-    _inputDevice->_axisStateMap[MOUSE_AXIS_Y_NEG] = (currentMove.y() > 0 ? currentMove.y() : 0.0f);
+        _inputDevice->_axisStateMap[MOUSE_AXIS_X_POS] = (currentMove.x() > 0 ? currentMove.x() : 0.0f);
+        _inputDevice->_axisStateMap[MOUSE_AXIS_X_NEG] = (currentMove.x() < 0 ? -currentMove.x() : 0.0f);
+        // Y mouse is inverted positive is pointing up the screen
+        _inputDevice->_axisStateMap[MOUSE_AXIS_Y_POS] = (currentMove.y() < 0 ? -currentMove.y() : 0.0f);
+        _inputDevice->_axisStateMap[MOUSE_AXIS_Y_NEG] = (currentMove.y() > 0 ? currentMove.y() : 0.0f);
 
-    // FIXME - this has the characteristic that it will show large jumps when you move the cursor
-    // outside of the application window, because we don't get MouseEvents when the cursor is outside
-    // of the application window.
-    _lastCursor = currentPos;
-    _mouseMoved = true;
+        // FIXME - this has the characteristic that it will show large jumps when you move the cursor
+        // outside of the application window, because we don't get MouseEvents when the cursor is outside
+        // of the application window.
+        _lastCursor = currentPos;
+        _mouseMoved = true;
 
-    eraseMouseClicked();
+        eraseMouseClicked();
+    }
 }
 
 void KeyboardMouseDevice::wheelEvent(QWheelEvent* event) {

libraries/input-plugins/src/input-plugins/KeyboardMouseDevice.h

@@ -85,6 +85,8 @@ public:
     void touchUpdateEvent(const QTouchEvent* event);
 
     void wheelEvent(QWheelEvent* event);
+
+    static void enableMouse(bool enableMouse) { _enableMouse = enableMouse; }
     
     static const QString NAME;
 
@@ -123,6 +125,8 @@ protected:
     bool _isTouching = false;
     std::chrono::high_resolution_clock _clock;
     std::chrono::high_resolution_clock::time_point _lastTouchTime;
+
+    static bool _enableMouse;
 };
 
 #endif // hifi_KeyboardMouseDevice_h

libraries/input-plugins/src/input-plugins/TouchscreenDevice.cpp

@@ -0,0 +1,119 @@
+//
+//  TouchscreenDevice.cpp
+//  input-plugins/src/input-plugins
+//
+//  Created by Triplelexx on 01/31/16.
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "TouchscreenDevice.h"
+#include "KeyboardMouseDevice.h"
+
+#include <QtGui/QTouchEvent>
+#include <QGestureEvent>
+#include <QGuiApplication>
+#include <QScreen>
+
+#include <controllers/UserInputMapper.h>
+#include <PathUtils.h>
+#include <NumericalConstants.h>
+
+const QString TouchscreenDevice::NAME = "Touchscreen";
+
+void TouchscreenDevice::pluginUpdate(float deltaTime, bool jointsCaptured) { 
+    _inputDevice->update(deltaTime, jointsCaptured);
+
+    // at DPI 100 use these arbitrary values to divide dragging distance 
+    static const float DPI_SCALE_X = glm::clamp((float)(qApp->primaryScreen()->physicalDotsPerInchX() / 100.0f), 1.0f, 10.0f)
+                       * 600.0f;
+    static const float DPI_SCALE_Y = glm::clamp((float)(qApp->primaryScreen()->physicalDotsPerInchY() / 100.0f), 1.0f, 10.0f)
+                       * 200.0f;
+
+    float distanceScaleX, distanceScaleY;
+    if (_touchPointCount == 1) {
+        if (_firstTouchVec.x < _currentTouchVec.x) {
+            distanceScaleX = (_currentTouchVec.x - _firstTouchVec.x) / DPI_SCALE_X;
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_AXIS_X_POS).getChannel()] = distanceScaleX;
+        } else if (_firstTouchVec.x > _currentTouchVec.x) {
+            distanceScaleX = (_firstTouchVec.x - _currentTouchVec.x) / DPI_SCALE_X;
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_AXIS_X_NEG).getChannel()] = distanceScaleX;
+        }
+        // Y axis is inverted, positive is pointing up the screen
+        if (_firstTouchVec.y > _currentTouchVec.y) {
+            distanceScaleY = (_firstTouchVec.y - _currentTouchVec.y) / DPI_SCALE_Y;
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_AXIS_Y_POS).getChannel()] = distanceScaleY;
+        } else if (_firstTouchVec.y < _currentTouchVec.y) {
+            distanceScaleY = (_currentTouchVec.y - _firstTouchVec.y) / DPI_SCALE_Y;
+            _inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_AXIS_Y_NEG).getChannel()] = distanceScaleY;
+        }
+    }
+}
+
+void TouchscreenDevice::InputDevice::update(float deltaTime, bool jointsCaptured) {
+    _axisStateMap.clear();
+}
+
+void TouchscreenDevice::InputDevice::focusOutEvent() {
+}
+
+void TouchscreenDevice::touchBeginEvent(const QTouchEvent* event) {
+    const QTouchEvent::TouchPoint& point = event->touchPoints().at(0);
+    _firstTouchVec = glm::vec2(point.pos().x(), point.pos().y());
+    KeyboardMouseDevice::enableMouse(false);
+}
+
+void TouchscreenDevice::touchEndEvent(const QTouchEvent* event) {
+    _touchPointCount = 0;
+    KeyboardMouseDevice::enableMouse(true);
+}
+
+void TouchscreenDevice::touchUpdateEvent(const QTouchEvent* event) {
+    const QTouchEvent::TouchPoint& point = event->touchPoints().at(0);
+    _currentTouchVec = glm::vec2(point.pos().x(), point.pos().y());
+    _touchPointCount = event->touchPoints().count();
+}
+
+void TouchscreenDevice::touchGestureEvent(const QGestureEvent* event) {
+	// pinch gesture
+	if (QGesture* gesture = event->gesture(Qt::PinchGesture)) {
+		QPinchGesture* pinch = static_cast<QPinchGesture*>(gesture);
+		qreal scaleFactor = pinch->totalScaleFactor();
+		if (scaleFactor > _lastPinchScale && scaleFactor != 0) {
+			_inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_GESTURE_PINCH_POS).getChannel()] = 1.0f;
+		} else if (scaleFactor != 0) {
+			_inputDevice->_axisStateMap[_inputDevice->makeInput(TOUCH_GESTURE_PINCH_NEG).getChannel()] = 1.0f;
+		}
+		_lastPinchScale = scaleFactor;
+	}
+}
+
+controller::Input TouchscreenDevice::InputDevice::makeInput(TouchscreenDevice::TouchAxisChannel axis) const {
+    return controller::Input(_deviceID, axis, controller::ChannelType::AXIS);
+}
+
+controller::Input TouchscreenDevice::InputDevice::makeInput(TouchscreenDevice::TouchGestureAxisChannel gesture) const {
+	return controller::Input(_deviceID, gesture, controller::ChannelType::AXIS);
+}
+
+controller::Input::NamedVector TouchscreenDevice::InputDevice::getAvailableInputs() const {
+    using namespace controller;
+    static QVector<Input::NamedPair> availableInputs;
+    static std::once_flag once;
+    std::call_once(once, [&] {
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_AXIS_X_POS), "DragRight"));
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_AXIS_X_NEG), "DragLeft"));
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_AXIS_Y_POS), "DragUp"));
+        availableInputs.append(Input::NamedPair(makeInput(TOUCH_AXIS_Y_NEG), "DragDown"));
+
+		availableInputs.append(Input::NamedPair(makeInput(TOUCH_GESTURE_PINCH_POS), "GesturePinchOut"));
+		availableInputs.append(Input::NamedPair(makeInput(TOUCH_GESTURE_PINCH_NEG), "GesturePinchIn"));
+    });
+    return availableInputs;
+}
+
+QString TouchscreenDevice::InputDevice::getDefaultMappingConfig() const {
+    static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/touchscreen.json";
+    return MAPPING_JSON;
+}

libraries/input-plugins/src/input-plugins/TouchscreenDevice.h

@@ -0,0 +1,81 @@
+//
+//  TouchscreenDevice.h
+//  input-plugins/src/input-plugins
+//
+//  Created by Triplelexx on 1/31/16.
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_TouchscreenDevice_h
+#define hifi_TouchscreenDevice_h
+
+#include <controllers/InputDevice.h>
+#include "InputPlugin.h"
+
+class QTouchEvent;
+class QGestureEvent;
+
+class TouchscreenDevice : public InputPlugin {
+    Q_OBJECT
+public:
+        
+    enum TouchAxisChannel {
+        TOUCH_AXIS_X_POS = 0,
+        TOUCH_AXIS_X_NEG,
+        TOUCH_AXIS_Y_POS,
+        TOUCH_AXIS_Y_NEG,
+    };
+
+	enum TouchGestureAxisChannel {
+        TOUCH_GESTURE_PINCH_POS = TOUCH_AXIS_Y_NEG + 1,
+		TOUCH_GESTURE_PINCH_NEG,
+	};
+
+    // Plugin functions
+    virtual bool isSupported() const override { return true; }
+    virtual bool isJointController() const override { return false; }
+    virtual const QString& getName() const override { return NAME; }
+
+    virtual void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); }
+    virtual void pluginUpdate(float deltaTime, bool jointsCaptured) override;
+
+    void touchBeginEvent(const QTouchEvent* event);
+    void touchEndEvent(const QTouchEvent* event);
+    void touchUpdateEvent(const QTouchEvent* event);
+	void touchGestureEvent(const QGestureEvent* event);
+    
+    static const QString NAME;
+
+protected:
+
+    class InputDevice : public controller::InputDevice {
+    public:
+        InputDevice() : controller::InputDevice("Touchscreen") {}
+    private:
+        // Device functions
+        virtual controller::Input::NamedVector getAvailableInputs() const override;
+        virtual QString getDefaultMappingConfig() const override;
+        virtual void update(float deltaTime, bool jointsCaptured) override;
+        virtual void focusOutEvent() override;
+
+        controller::Input makeInput(TouchAxisChannel axis) const;
+		controller::Input makeInput(TouchGestureAxisChannel gesture) const;
+
+        friend class TouchscreenDevice;
+    };
+
+public:
+    const std::shared_ptr<InputDevice>& getInputDevice() const { return _inputDevice; }
+
+protected:
+    qreal _lastPinchScale;
+    glm::vec2 _firstTouchVec;
+    glm::vec2 _currentTouchVec;
+    int _touchPointCount;
+    std::shared_ptr<InputDevice> _inputDevice { std::make_shared<InputDevice>() };
+};
+
+#endif // hifi_TouchscreenDevice_h

tests/controllers/src/main.cpp

@@ -38,6 +38,7 @@
 #include <plugins/PluginManager.h>
 #include <input-plugins/InputPlugin.h>
 #include <input-plugins/KeyboardMouseDevice.h>
+#include <input-plugins/TouchscreenDevice.h>
 #include <controllers/ScriptingInterface.h>
 
 #include <DependencyManager.h>
@@ -153,6 +154,9 @@ int main(int argc, char** argv) {
             if (name == KeyboardMouseDevice::NAME) {
                 userInputMapper->registerDevice(std::dynamic_pointer_cast<KeyboardMouseDevice>(inputPlugin)->getInputDevice());
             }
+            if (name == TouchscreenDevice::NAME) {
+                userInputMapper->registerDevice(std::dynamic_pointer_cast<TouchscreenDevice>(inputPlugin)->getInputDevice());
+            }
             inputPlugin->pluginUpdate(0, calibrationData, false);
         }
         rootContext->setContextProperty("Controllers", new MyControllerScriptingInterface());