Merge pull request #350 from ctrlaltdavid/feature/vive-pro-eye-tracking

Vive PRO Eye support

README.md

@@ -14,21 +14,29 @@ Vircadia is a 3D social software project seeking to incrementally bring about a
 
 [For Windows](https://github.com/kasenvr/project-athena/blob/master/BUILD_WIN.md)
 
+[For Mac](https://github.com/kasenvr/project-athena/blob/master/BUILD_OSX.md)
+
 [For Linux](https://github.com/kasenvr/project-athena/blob/master/BUILD_LINUX.md)
 
-[For Linux - Athena Builder](https://github.com/kasenvr/vircadia-builder)
+[For Linux - Vircadia Builder](https://github.com/kasenvr/vircadia-builder)
 
 ### How to deploy a Server
 
-[For Windows and Linux](https://vircadia.com/download-vircadia/#server)
+[For Windows and Linux](https://vircadia.com/deploy-a-server/)
 
 ### How to build a Server
 
-[For Linux - Athena Builder](https://github.com/kasenvr/vircadia-builder)
+[For Linux - Vircadia Builder](https://github.com/kasenvr/vircadia-builder)
+
+### How to generate an Installer
+
+[For Windows](https://github.com/kasenvr/project-athena/blob/master/INSTALL.md)
+
+[For Linux - AppImage - Vircadia Builder](https://github.com/kasenvr/vircadia-builder/blob/master/README.md#building-appimages)
 
 ### Boot to Metaverse: The Goal
 
-Having a place to experience adventure, a place to relax with calm breath, that's a world to live in. An engine to support infinite combinations and possibilities of worlds without censorship and interruption, that's a metaverse. Finding a way to make infinite realities our reality, that's the dream.
+Having a place to experience adventure, a place to relax with calm breath, that's a world to live in. An engine to support infinite combinations and possibilities of worlds without censorship and interruption, that's a metaverse. Finding a way to make infinite realities our reality is the dream.
 
 ### Boot to Metaverse: The Technicals
 

cmake/ports/aristo/portfile.cmake

@@ -2,10 +2,12 @@ include(vcpkg_common_functions)
 set(ARISTO_VERSION 0.8.1)
 set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src)
 
+file(READ "${VCPKG_ROOT_DIR}/_env/EXTERNAL_BUILD_ASSETS.txt" EXTERNAL_BUILD_ASSETS)
+
 if (WIN32)
     vcpkg_download_distfile(
         ARISTO_SOURCE_ARCHIVE
-        URLS https://athena-public.s3.amazonaws.com/seth/aristo-0.8.1-windows.zip
+        URLS "${EXTERNAL_BUILD_ASSETS}/seth/aristo-0.8.1-windows.zip"
         SHA512 05179c63b72a1c9f5be8a7a2b7389025da683400dbf819e5a6199dd6473c56774d2885182dc5a11cb6324058d228a4ead832222e8b3e1bebaa4c61982e85f0a8
         FILENAME aristo-0.8.1-windows.zip
     )

cmake/ports/sranipal/CONTROL

@@ -1,3 +1,3 @@
 Source: sranipal
 Version: 1.1.0.1
-Description: SRanipal
+Description: super reality animation pal!

cmake/ports/sranipal/portfile.cmake

@@ -2,10 +2,12 @@ include(vcpkg_common_functions)
 set(SRANIPAL_VERSION 1.1.0.1)
 set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src)
 
+file(READ "${VCPKG_ROOT_DIR}/_env/EXTERNAL_BUILD_ASSETS.txt" EXTERNAL_BUILD_ASSETS)
+
 if (WIN32)
     vcpkg_download_distfile(
         SRANIPAL_SOURCE_ARCHIVE
-        URLS https://athena-public.s3.amazonaws.com/seth/sranipal-1.1.0.1-windows.zip
+        URLS "${EXTERNAL_BUILD_ASSETS}/seth/sranipal-1.1.0.1-windows.zip"
         SHA512 b09ce012abe4e3c71e8e69626bdd7823ff6576601a821ab365275f2764406a3e5f7b65fcf2eb1d0962eff31eb5958a148b00901f67c229dc6ace56eb5e6c9e1b
         FILENAME sranipal-1.1.0.1-windows.zip
     )

interface/src/Application.h

@@ -223,9 +223,7 @@ public:
     bool getPreferStylusOverLaser() { return _preferStylusOverLaserSetting.get(); }
     void setPreferStylusOverLaser(bool value);
 
-    // FIXME: Remove setting completely or make available through JavaScript API?
+    bool getPreferAvatarFingerOverStylus() { return _preferAvatarFingerOverStylusSetting.get(); }
-    //bool getPreferAvatarFingerOverStylus() { return _preferAvatarFingerOverStylusSetting.get(); }
-    bool getPreferAvatarFingerOverStylus() { return false; }
     void setPreferAvatarFingerOverStylus(bool value);
 
     bool getMiniTabletEnabled() { return _miniTabletEnabledSetting.get(); }

interface/src/ui/PreferencesDialog.cpp

@@ -223,13 +223,11 @@ void setupPreferences() {
         preferences->addPreference(preference);
     }
 
-    /*
-    // FIXME: Remove setting completely or make available through JavaScript API?
     {
         auto getter = []()->bool { return qApp->getPreferAvatarFingerOverStylus(); };
         auto setter = [](bool value) { qApp->setPreferAvatarFingerOverStylus(value); };
         preferences->addPreference(new CheckPreference(UI_CATEGORY, "Prefer Avatar Finger Over Stylus", getter, setter));
-        }*/
+    }
 
     // Snapshots
     static const QString SNAPSHOTS { "Snapshots" };

plugins/hifiLeapMotion/src/LeapMotionPlugin.cpp

@@ -281,6 +281,7 @@ void LeapMotionPlugin::InputDevice::update(float deltaTime, const controller::In
 
         glm::vec3 pos;
         glm::quat rot;
+        glm::quat prevRot;
         if (_isLeapOnHMD) {
             auto jointPosition = joints[i].position;
             const glm::vec3 HMD_EYE_TO_LEAP_OFFSET = glm::vec3(0.0f, 0.0f, -0.09f);  // Eyes to surface of Leap Motion.
@@ -291,17 +292,24 @@ void LeapMotionPlugin::InputDevice::update(float deltaTime, const controller::In
             glm::quat jointOrientation = joints[i].orientation;
             jointOrientation = glm::quat(jointOrientation.w, -jointOrientation.x, -jointOrientation.z, -jointOrientation.y);
             rot = controllerToAvatarRotation * hmdSensorOrientation * jointOrientation;
+
+            glm::quat prevJointOrientation = prevJoints[i].orientation;
+            prevJointOrientation =
+                glm::quat(prevJointOrientation.w, -prevJointOrientation.x, -prevJointOrientation.z, -prevJointOrientation.y);
+            prevRot = controllerToAvatarRotation * hmdSensorOrientation * prevJointOrientation;
+
         } else {
             pos = controllerToAvatarRotation * (joints[i].position - leapMotionOffset);
             const glm::quat ZERO_HAND_ORIENTATION = glm::quat(glm::vec3(PI_OVER_TWO, PI, 0.0f));
             rot = controllerToAvatarRotation * joints[i].orientation * ZERO_HAND_ORIENTATION;
+            prevRot = controllerToAvatarRotation * prevJoints[i].orientation * ZERO_HAND_ORIENTATION;
         }
 
         glm::vec3 linearVelocity, angularVelocity;
         if (i < prevJoints.size()) {
             linearVelocity = (pos - (prevJoints[i].position * METERS_PER_CENTIMETER)) / deltaTime;  // m/s
             // quat log imaginary part points along the axis of rotation, with length of one half the angle of rotation.
-            glm::quat d = glm::log(rot * glm::inverse(prevJoints[i].orientation));
+            glm::quat d = glm::log(rot * glm::inverse(prevRot));
             angularVelocity = glm::vec3(d.x, d.y, d.z) / (0.5f * deltaTime); // radians/s
         }
 

plugins/openvr/CMakeLists.txt

@@ -15,5 +15,7 @@ if (WIN32 AND (NOT USE_GLES))
     include_hifi_library_headers(octree)
 
     target_openvr()
+    target_sranipal()
+    target_aristo()
     target_link_libraries(${TARGET_NAME} Winmm.lib)
 endif()

plugins/openvr/src/ViveControllerManager.cpp

@@ -13,6 +13,21 @@
 #include <algorithm>
 #include <string>
 
+#ifdef _WIN32
+#pragma warning( push )
+#pragma warning( disable : 4091 )
+#pragma warning( disable : 4334 )
+#endif
+
+#include <SRanipal.h>
+#include <SRanipal_Eye.h>
+#include <SRanipal_Enums.h>
+#include <interface_gesture.hpp>
+
+#ifdef _WIN32
+#pragma warning( pop )
+#endif
+
 #include <PerfStat.h>
 #include <PathUtils.h>
 #include <GeometryCache.h>
@@ -37,6 +52,8 @@
 #include <Plugins/InputConfiguration.h>
 #include <controllers/StandardControls.h>
 
+#include "OpenVrDisplayPlugin.h"
+
 extern PoseData _nextSimPoseData;
 
 vr::IVRSystem* acquireOpenVrSystem();
@@ -62,6 +79,32 @@ static const int SECOND_FOOT = 1;
 static const int HIP = 2;
 static const int CHEST = 3;
 
+enum ViveHandJointIndex {
+    HAND = 0,
+    THUMB_1,
+    THUMB_2,
+    THUMB_3,
+    THUMB_4,
+    INDEX_1,
+    INDEX_2,
+    INDEX_3,
+    INDEX_4,
+    MIDDLE_1,
+    MIDDLE_2,
+    MIDDLE_3,
+    MIDDLE_4,
+    RING_1,
+    RING_2,
+    RING_3,
+    RING_4,
+    PINKY_1,
+    PINKY_2,
+    PINKY_3,
+    PINKY_4,
+
+    Size
+};
+
 const char* ViveControllerManager::NAME { "OpenVR" };
 
 const std::map<vr::ETrackingResult, QString> TRACKING_RESULT_TO_STRING = {
@@ -130,6 +173,51 @@ static glm::mat4 calculateResetMat() {
     return glm::mat4();
 }
 
+class ViveProEyeReadThread : public QThread {
+public:
+    ViveProEyeReadThread() {
+        setObjectName("OpenVR ViveProEye Read Thread");
+    }
+    void run() override {
+        while (!quit) {
+            ViveSR::anipal::Eye::EyeData eyeData;
+            int result = ViveSR::anipal::Eye::GetEyeData(&eyeData);
+            {
+                QMutexLocker locker(&eyeDataMutex);
+                eyeDataBuffer.getEyeDataResult = result;
+                if (result == ViveSR::Error::WORK) {
+                    uint64_t leftValids = eyeData.verbose_data.left.eye_data_validata_bit_mask;
+                    uint64_t rightValids = eyeData.verbose_data.right.eye_data_validata_bit_mask;
+
+                    eyeDataBuffer.leftDirectionValid =
+                        (leftValids & (uint64_t)ViveSR::anipal::Eye::SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY) > (uint64_t)0;
+                    eyeDataBuffer.rightDirectionValid =
+                        (rightValids & (uint64_t)ViveSR::anipal::Eye::SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY) > (uint64_t)0;
+                    eyeDataBuffer.leftOpennessValid =
+                        (leftValids & (uint64_t)ViveSR::anipal::Eye::SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY) > (uint64_t)0;
+                    eyeDataBuffer.rightOpennessValid =
+                        (rightValids & (uint64_t)ViveSR::anipal::Eye::SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY) > (uint64_t)0;
+
+                    float *leftGaze = eyeData.verbose_data.left.gaze_direction_normalized.elem_;
+                    float *rightGaze = eyeData.verbose_data.right.gaze_direction_normalized.elem_;
+                    eyeDataBuffer.leftEyeGaze = glm::vec3(leftGaze[0], leftGaze[1], leftGaze[2]);
+                    eyeDataBuffer.rightEyeGaze = glm::vec3(rightGaze[0], rightGaze[1], rightGaze[2]);
+
+                    eyeDataBuffer.leftEyeOpenness = eyeData.verbose_data.left.eye_openness;
+                    eyeDataBuffer.rightEyeOpenness = eyeData.verbose_data.right.eye_openness;
+                }
+            }
+        }
+    }
+
+    bool quit { false };
+
+    // mutex and buffer for moving data from this thread to the other one
+    QMutex eyeDataMutex;
+    EyeDataBuffer eyeDataBuffer;
+};
+
+
 static QString outOfRangeDataStrategyToString(ViveControllerManager::OutOfRangeDataStrategy strategy) {
     switch (strategy) {
     default:
@@ -211,6 +299,81 @@ QString ViveControllerManager::configurationLayout() {
     return OPENVR_LAYOUT;
 }
 
+bool isDeviceIndexActive(vr::IVRSystem*& system, uint32_t deviceIndex) {
+    if (!system) {
+        return false;
+    }
+    if (deviceIndex != vr::k_unTrackedDeviceIndexInvalid &&
+        system->GetTrackedDeviceClass(deviceIndex) == vr::TrackedDeviceClass_Controller &&
+        system->IsTrackedDeviceConnected(deviceIndex)) {
+        vr::EDeviceActivityLevel activityLevel = system->GetTrackedDeviceActivityLevel(deviceIndex);
+        if (activityLevel == vr::k_EDeviceActivityLevel_UserInteraction) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool isHandControllerActive(vr::IVRSystem*& system, vr::ETrackedControllerRole deviceRole) {
+    if (!system) {
+        return false;
+    }
+    auto deviceIndex = system->GetTrackedDeviceIndexForControllerRole(deviceRole);
+    return isDeviceIndexActive(system, deviceIndex);
+}
+
+bool areBothHandControllersActive(vr::IVRSystem*& system) {
+    return
+        isHandControllerActive(system, vr::TrackedControllerRole_LeftHand) &&
+        isHandControllerActive(system, vr::TrackedControllerRole_RightHand);
+}
+
+
+void ViveControllerManager::enableGestureDetection() {
+    if (_viveCameraHandTracker) {
+        return;
+    }
+    if (!ViveSR::anipal::Eye::IsViveProEye()) {
+        return;
+    }
+
+// #define HAND_TRACKER_USE_EXTERNAL_TRANSFORM 1
+
+#ifdef HAND_TRACKER_USE_EXTERNAL_TRANSFORM
+    UseExternalTransform(true); // camera hand tracker results are in HMD frame
+#else
+    UseExternalTransform(false); // camera hand tracker results are in sensor frame
+#endif
+    GestureOption options; // defaults are GestureBackendAuto and GestureModeSkeleton
+    GestureFailure gestureFailure = StartGestureDetection(&options);
+    switch (gestureFailure) {
+        case GestureFailureNone:
+            qDebug() << "StartGestureDetection success";
+            _viveCameraHandTracker = true;
+            break;
+        case GestureFailureOpenCL:
+            qDebug() << "StartGestureDetection (Only on Windows) OpenCL is not supported on the machine";
+            break;
+        case GestureFailureCamera:
+            qDebug() << "StartGestureDetection Start camera failed";
+            break;
+        case GestureFailureInternal:
+            qDebug() << "StartGestureDetection Internal errors";
+            break;
+        case GestureFailureCPUOnPC:
+            qDebug() << "StartGestureDetection CPU backend is not supported on Windows";
+            break;
+    }
+}
+
+void ViveControllerManager::disableGestureDetection() {
+    if (!_viveCameraHandTracker) {
+        return;
+    }
+    StopGestureDetection();
+    _viveCameraHandTracker = false;
+}
+
 bool ViveControllerManager::activate() {
     InputPlugin::activate();
 
@@ -230,6 +393,28 @@ bool ViveControllerManager::activate() {
     auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
     userInputMapper->registerDevice(_inputDevice);
     _registeredWithInputMapper = true;
+
+    if (ViveSR::anipal::Eye::IsViveProEye()) {
+        qDebug() << "Vive Pro eye-tracking detected";
+
+        int error = ViveSR::anipal::Initial(ViveSR::anipal::Eye::ANIPAL_TYPE_EYE, NULL);
+        if (error == ViveSR::Error::WORK) {
+            _viveProEye = true;
+            qDebug() << "Successfully initialize Eye engine.";
+        } else if (error == ViveSR::Error::RUNTIME_NOT_FOUND) {
+            _viveProEye = false;
+            qDebug() << "please follows SRanipal SDK guide to install SR_Runtime first";
+        } else {
+            _viveProEye = false;
+            qDebug() << "Failed to initialize Eye engine. please refer to ViveSR error code:" << error;
+        }
+
+        if (_viveProEye) {
+            _viveProEyeReadThread = std::make_shared<ViveProEyeReadThread>();
+            _viveProEyeReadThread->start(QThread::HighPriority);
+        }
+    }
+
     return true;
 }
 
@@ -251,6 +436,13 @@ void ViveControllerManager::deactivate() {
     userInputMapper->removeDevice(_inputDevice->_deviceID);
     _registeredWithInputMapper = false;
 
+    if (_viveProEyeReadThread) {
+        _viveProEyeReadThread->quit = true;
+        _viveProEyeReadThread->wait();
+        _viveProEyeReadThread = nullptr;
+        ViveSR::anipal::Release(ViveSR::anipal::Eye::ANIPAL_TYPE_EYE);
+    }
+
     saveSettings();
 }
 
@@ -262,6 +454,317 @@ bool ViveControllerManager::isHeadControllerMounted() const {
     return activityLevel == vr::k_EDeviceActivityLevel_UserInteraction;
 }
 
+void ViveControllerManager::invalidateEyeInputs() {
+    _inputDevice->_poseStateMap[controller::LEFT_EYE].valid = false;
+    _inputDevice->_poseStateMap[controller::RIGHT_EYE].valid = false;
+    _inputDevice->_axisStateMap[controller::EYEBLINK_L].valid = false;
+    _inputDevice->_axisStateMap[controller::EYEBLINK_R].valid = false;
+}
+
+
+void ViveControllerManager::updateEyeTracker(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
+    if (!isHeadControllerMounted()) {
+        invalidateEyeInputs();
+        return;
+    }
+
+    EyeDataBuffer eyeDataBuffer;
+    {
+        // GetEyeData takes around 4ms to finish, so we run it on a thread.
+        QMutexLocker locker(&_viveProEyeReadThread->eyeDataMutex);
+        memcpy(&eyeDataBuffer, &_viveProEyeReadThread->eyeDataBuffer, sizeof(eyeDataBuffer));
+    }
+
+    if (eyeDataBuffer.getEyeDataResult != ViveSR::Error::WORK) {
+        invalidateEyeInputs();
+        return;
+    }
+
+    // only update from buffer values if the new data is "valid"
+    if (!eyeDataBuffer.leftDirectionValid) {
+        eyeDataBuffer.leftEyeGaze = _prevEyeData.leftEyeGaze;
+        eyeDataBuffer.leftDirectionValid = _prevEyeData.leftDirectionValid;
+    }
+    if (!eyeDataBuffer.rightDirectionValid) {
+        eyeDataBuffer.rightEyeGaze = _prevEyeData.rightEyeGaze;
+        eyeDataBuffer.rightDirectionValid = _prevEyeData.rightDirectionValid;
+    }
+    if (!eyeDataBuffer.leftOpennessValid) {
+        eyeDataBuffer.leftEyeOpenness = _prevEyeData.leftEyeOpenness;
+        eyeDataBuffer.leftOpennessValid = _prevEyeData.leftOpennessValid;
+    }
+    if (!eyeDataBuffer.rightOpennessValid) {
+        eyeDataBuffer.rightEyeOpenness = _prevEyeData.rightEyeOpenness;
+        eyeDataBuffer.rightOpennessValid = _prevEyeData.rightOpennessValid;
+    }
+    _prevEyeData = eyeDataBuffer;
+
+    // transform data into what the controller system expects.
+
+    // in the data from sranipal, left=+x, up=+y, forward=+z
+    mat4 localLeftEyeMat = glm::lookAt(vec3(0.0f, 0.0f, 0.0f),
+                                       glm::vec3(eyeDataBuffer.leftEyeGaze[0],
+                                                 eyeDataBuffer.leftEyeGaze[1],
+                                                 -eyeDataBuffer.leftEyeGaze[2]),
+                                       vec3(0.0f, 1.0f, 0.0f));
+    quat localLeftEyeRot = glm::quat_cast(localLeftEyeMat);
+    quat avatarLeftEyeRot = _inputDevice->_poseStateMap[controller::HEAD].rotation * localLeftEyeRot;
+
+    mat4 localRightEyeMat = glm::lookAt(vec3(0.0f, 0.0f, 0.0f),
+                                        glm::vec3(eyeDataBuffer.rightEyeGaze[0],
+                                                  eyeDataBuffer.rightEyeGaze[1],
+                                                  -eyeDataBuffer.rightEyeGaze[2]),
+                                        vec3(0.0f, 1.0f, 0.0f));
+    quat localRightEyeRot = glm::quat_cast(localRightEyeMat);
+    quat avatarRightEyeRot = _inputDevice->_poseStateMap[controller::HEAD].rotation * localRightEyeRot;
+
+    // TODO -- figure out translations for eyes
+    if (eyeDataBuffer.leftDirectionValid) {
+        _inputDevice->_poseStateMap[controller::LEFT_EYE] = controller::Pose(glm::vec3(), avatarLeftEyeRot);
+        _inputDevice->_poseStateMap[controller::LEFT_EYE].valid = true;
+    } else {
+        _inputDevice->_poseStateMap[controller::LEFT_EYE].valid = false;
+    }
+    if (eyeDataBuffer.rightDirectionValid) {
+        _inputDevice->_poseStateMap[controller::RIGHT_EYE] = controller::Pose(glm::vec3(), avatarRightEyeRot);
+        _inputDevice->_poseStateMap[controller::RIGHT_EYE].valid = true;
+    } else {
+        _inputDevice->_poseStateMap[controller::RIGHT_EYE].valid = false;
+    }
+
+    quint64 now = usecTimestampNow();
+
+    // in hifi, 0 is open 1 is closed.  in SRanipal 1 is open, 0 is closed.
+    if (eyeDataBuffer.leftOpennessValid) {
+        _inputDevice->_axisStateMap[controller::EYEBLINK_L] =
+            controller::AxisValue(1.0f - eyeDataBuffer.leftEyeOpenness, now);
+    } else {
+        _inputDevice->_poseStateMap[controller::EYEBLINK_L].valid = false;
+    }
+    if (eyeDataBuffer.rightOpennessValid) {
+        _inputDevice->_axisStateMap[controller::EYEBLINK_R] =
+            controller::AxisValue(1.0f - eyeDataBuffer.rightEyeOpenness, now);
+    } else {
+        _inputDevice->_poseStateMap[controller::EYEBLINK_R].valid = false;
+    }
+}
+
+glm::vec3 ViveControllerManager::getRollingAverageHandPoint(int handIndex, int pointIndex) const {
+#if 0
+    return _handPoints[0][handIndex][pointIndex];
+#else
+    glm::vec3 result;
+    for (int s = 0; s < NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES; s++) {
+        result += _handPoints[s][handIndex][pointIndex];
+    }
+    return result / NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES;
+#endif
+}
+
+
+controller::Pose ViveControllerManager::trackedHandDataToPose(int hand, const glm::vec3& palmFacing,
+                                                              int nearHandPositionIndex, int farHandPositionIndex) {
+    glm::vec3 nearPoint = getRollingAverageHandPoint(hand, nearHandPositionIndex);
+
+    glm::quat poseRot;
+    if (nearHandPositionIndex != farHandPositionIndex) {
+        glm::vec3 farPoint = getRollingAverageHandPoint(hand, farHandPositionIndex);
+
+        glm::vec3 pointingDir = farPoint - nearPoint; // y axis
+        glm::vec3 otherAxis = glm::cross(pointingDir, palmFacing);
+
+        glm::mat4 rotMat;
+        rotMat = glm::mat4(glm::vec4(otherAxis, 0.0f),
+                           glm::vec4(pointingDir, 0.0f),
+                           glm::vec4(palmFacing * (hand == 0 ? 1.0f : -1.0f), 0.0f),
+                           glm::vec4(0.0f, 0.0f, 0.0f, 1.0f));
+        poseRot = glm::normalize(glmExtractRotation(rotMat));
+    }
+
+    if (!isNaN(poseRot)) {
+        controller::Pose pose(nearPoint, poseRot);
+        return pose;
+    } else {
+        controller::Pose pose;
+        pose.valid = false;
+        return pose;
+    }
+}
+
+
+void ViveControllerManager::trackFinger(int hand, int jointIndex1, int jointIndex2, int jointIndex3, int jointIndex4,
+                                        controller::StandardPoseChannel joint1, controller::StandardPoseChannel joint2,
+                                        controller::StandardPoseChannel joint3, controller::StandardPoseChannel joint4) {
+
+    glm::vec3 point1 = getRollingAverageHandPoint(hand, jointIndex1);
+    glm::vec3 point2 = getRollingAverageHandPoint(hand, jointIndex2);
+    glm::vec3 point3 = getRollingAverageHandPoint(hand, jointIndex3);
+    glm::vec3 point4 = getRollingAverageHandPoint(hand, jointIndex4);
+
+    glm::vec3 wristPos = getRollingAverageHandPoint(hand, ViveHandJointIndex::HAND);
+    glm::vec3 thumb2 = getRollingAverageHandPoint(hand, ViveHandJointIndex::THUMB_2);
+    glm::vec3 pinkie1 = getRollingAverageHandPoint(hand, ViveHandJointIndex::PINKY_1);
+
+    // 1st
+    glm::vec3 palmFacing = glm::normalize(glm::cross(pinkie1 - wristPos, thumb2 - wristPos));
+    glm::vec3 handForward = glm::normalize(point1 - wristPos);
+    glm::vec3 x = glm::normalize(glm::cross(palmFacing, handForward));
+    glm::vec3 y = glm::normalize(point2 - point1);
+    glm::vec3 z = (hand == 0) ? glm::cross(y, x) : glm::cross(x, y);
+    glm::mat4 rotMat1 = glm::mat4(glm::vec4(x, 0.0f),
+                                  glm::vec4(y, 0.0f),
+                                  glm::vec4(z, 0.0f),
+                                  glm::vec4(0.0f, 0.0f, 0.0f, 1.0f));
+    glm::quat rot1 = glm::normalize(glmExtractRotation(rotMat1));
+    if (!isNaN(rot1)) {
+        _inputDevice->_poseStateMap[joint1] = controller::Pose(point1, rot1);
+    }
+
+
+    // 2nd
+    glm::vec3 x2 = x; // glm::normalize(glm::cross(point3 - point2, point2 - point1));
+    glm::vec3 y2 = glm::normalize(point3 - point2);
+    glm::vec3 z2 = (hand == 0) ? glm::cross(y2, x2) : glm::cross(x2, y2);
+
+    glm::mat4 rotMat2 = glm::mat4(glm::vec4(x2, 0.0f),
+                                  glm::vec4(y2, 0.0f),
+                                  glm::vec4(z2, 0.0f),
+                                  glm::vec4(0.0f, 0.0f, 0.0f, 1.0f));
+    glm::quat rot2 = glm::normalize(glmExtractRotation(rotMat2));
+    if (!isNaN(rot2)) {
+        _inputDevice->_poseStateMap[joint2] = controller::Pose(point2, rot2);
+    }
+
+
+    // 3rd
+    glm::vec3 x3 = x; // glm::normalize(glm::cross(point4 - point3, point3 - point1));
+    glm::vec3 y3 = glm::normalize(point4 - point3);
+    glm::vec3 z3 = (hand == 0) ? glm::cross(y3, x3) : glm::cross(x3, y3);
+
+    glm::mat4 rotMat3 = glm::mat4(glm::vec4(x3, 0.0f),
+                                  glm::vec4(y3, 0.0f),
+                                  glm::vec4(z3, 0.0f),
+                                  glm::vec4(0.0f, 0.0f, 0.0f, 1.0f));
+    glm::quat rot3 = glm::normalize(glmExtractRotation(rotMat3));
+    if (!isNaN(rot3)) {
+        _inputDevice->_poseStateMap[joint3] = controller::Pose(point3, rot3);
+    }
+
+
+    // 4th
+    glm::quat rot4 = rot3;
+    if (!isNaN(rot4)) {
+        _inputDevice->_poseStateMap[joint4] = controller::Pose(point4, rot4);
+    }
+}
+
+
+void ViveControllerManager::updateCameraHandTracker(float deltaTime,
+                                                    const controller::InputCalibrationData& inputCalibrationData) {
+
+    if (areBothHandControllersActive(_system)) {
+        // if both hand-controllers are in use, don't do camera hand tracking
+        disableGestureDetection();
+    } else {
+        enableGestureDetection();
+    }
+
+    if (!_viveCameraHandTracker) {
+        return;
+    }
+
+    const GestureResult* results = NULL;
+    int handTrackerFrameIndex { -1 };
+    int resultsHandCount = GetGestureResult(&results, &handTrackerFrameIndex);
+
+    // FIXME: Why the commented-out condition?
+    if (handTrackerFrameIndex >= 0 /* && handTrackerFrameIndex != _lastHandTrackerFrameIndex */) {
+#ifdef HAND_TRACKER_USE_EXTERNAL_TRANSFORM
+        glm::mat4 trackedHandToAvatar =
+            glm::inverse(inputCalibrationData.avatarMat) *
+            inputCalibrationData.sensorToWorldMat *
+            inputCalibrationData.hmdSensorMat;
+        // glm::mat4 trackedHandToAvatar = _inputDevice->_poseStateMap[controller::HEAD].getMatrix() * Matrices::Y_180;
+#else
+        DisplayPluginPointer displayPlugin = _container->getActiveDisplayPlugin();
+        std::shared_ptr<OpenVrDisplayPlugin> openVRDisplayPlugin =
+            std::dynamic_pointer_cast<OpenVrDisplayPlugin>(displayPlugin);
+        glm::mat4 sensorResetMatrix;
+        if (openVRDisplayPlugin) {
+            sensorResetMatrix = openVRDisplayPlugin->getSensorResetMatrix();
+        }
+
+        glm::mat4 trackedHandToAvatar =
+            glm::inverse(inputCalibrationData.avatarMat) *
+            inputCalibrationData.sensorToWorldMat *
+            sensorResetMatrix;
+#endif
+
+        // roll all the old points in the rolling average
+        memmove(&(_handPoints[1]),
+                &(_handPoints[0]),
+                sizeof(_handPoints[0]) * (NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES - 1));
+
+        for (int handIndex = 0; handIndex < resultsHandCount; handIndex++) {
+            bool isLeftHand = results[handIndex].isLeft;
+
+            vr::ETrackedControllerRole controllerRole =
+                isLeftHand ? vr::TrackedControllerRole_LeftHand : vr::TrackedControllerRole_RightHand;
+            if (isHandControllerActive(_system, controllerRole)) {
+                continue; // if the controller for this hand is tracked, ignore camera hand tracking
+            }
+
+            int hand =  isLeftHand ? 0 : 1;
+            for (int pointIndex = 0; pointIndex < NUMBER_OF_HAND_POINTS; pointIndex++) {
+                glm::vec3 pos(results[handIndex].points[3 * pointIndex],
+                              results[handIndex].points[3 * pointIndex + 1],
+                              -results[handIndex].points[3 * pointIndex + 2]);
+                _handPoints[0][hand][pointIndex] = transformPoint(trackedHandToAvatar, pos);
+            }
+
+            glm::vec3 wristPos = getRollingAverageHandPoint(hand, ViveHandJointIndex::HAND);
+            glm::vec3 thumb2 = getRollingAverageHandPoint(hand, ViveHandJointIndex::THUMB_2);
+            glm::vec3 pinkie1 = getRollingAverageHandPoint(hand, ViveHandJointIndex::PINKY_1);
+            glm::vec3 palmFacing = glm::cross(pinkie1 - wristPos, thumb2 - wristPos); // z axis
+
+            _inputDevice->_poseStateMap[isLeftHand ? controller::LEFT_HAND : controller::RIGHT_HAND] =
+                trackedHandDataToPose(hand, palmFacing, ViveHandJointIndex::HAND, ViveHandJointIndex::MIDDLE_1);
+            trackFinger(hand, ViveHandJointIndex::THUMB_1, ViveHandJointIndex::THUMB_2, ViveHandJointIndex::THUMB_3,
+                        ViveHandJointIndex::THUMB_4,
+                        isLeftHand ? controller::LEFT_HAND_THUMB1 : controller::RIGHT_HAND_THUMB1,
+                        isLeftHand ? controller::LEFT_HAND_THUMB2 : controller::RIGHT_HAND_THUMB2,
+                        isLeftHand ? controller::LEFT_HAND_THUMB3 : controller::RIGHT_HAND_THUMB3,
+                        isLeftHand ? controller::LEFT_HAND_THUMB4 : controller::RIGHT_HAND_THUMB4);
+            trackFinger(hand, ViveHandJointIndex::INDEX_1, ViveHandJointIndex::INDEX_2, ViveHandJointIndex::INDEX_3,
+                        ViveHandJointIndex::INDEX_4,
+                        isLeftHand ? controller::LEFT_HAND_INDEX1 : controller::RIGHT_HAND_INDEX1,
+                        isLeftHand ? controller::LEFT_HAND_INDEX2 : controller::RIGHT_HAND_INDEX2,
+                        isLeftHand ? controller::LEFT_HAND_INDEX3 : controller::RIGHT_HAND_INDEX3,
+                        isLeftHand ? controller::LEFT_HAND_INDEX4 : controller::RIGHT_HAND_INDEX4);
+            trackFinger(hand, ViveHandJointIndex::MIDDLE_1, ViveHandJointIndex::MIDDLE_2, ViveHandJointIndex::MIDDLE_3,
+                        ViveHandJointIndex::MIDDLE_4,
+                        isLeftHand ? controller::LEFT_HAND_MIDDLE1 : controller::RIGHT_HAND_MIDDLE1,
+                        isLeftHand ? controller::LEFT_HAND_MIDDLE2 : controller::RIGHT_HAND_MIDDLE2,
+                        isLeftHand ? controller::LEFT_HAND_MIDDLE3 : controller::RIGHT_HAND_MIDDLE3,
+                        isLeftHand ? controller::LEFT_HAND_MIDDLE4 : controller::RIGHT_HAND_MIDDLE4);
+            trackFinger(hand, ViveHandJointIndex::RING_1, ViveHandJointIndex::RING_2, ViveHandJointIndex::RING_3,
+                        ViveHandJointIndex::RING_4,
+                        isLeftHand ? controller::LEFT_HAND_RING1 : controller::RIGHT_HAND_RING1,
+                        isLeftHand ? controller::LEFT_HAND_RING2 : controller::RIGHT_HAND_RING2,
+                        isLeftHand ? controller::LEFT_HAND_RING3 : controller::RIGHT_HAND_RING3,
+                        isLeftHand ? controller::LEFT_HAND_RING4 : controller::RIGHT_HAND_RING4);
+            trackFinger(hand, ViveHandJointIndex::PINKY_1, ViveHandJointIndex::PINKY_2, ViveHandJointIndex::PINKY_3,
+                        ViveHandJointIndex::PINKY_4,
+                        isLeftHand ? controller::LEFT_HAND_PINKY1 : controller::RIGHT_HAND_PINKY1,
+                        isLeftHand ? controller::LEFT_HAND_PINKY2 : controller::RIGHT_HAND_PINKY2,
+                        isLeftHand ? controller::LEFT_HAND_PINKY3 : controller::RIGHT_HAND_PINKY3,
+                        isLeftHand ? controller::LEFT_HAND_PINKY4 : controller::RIGHT_HAND_PINKY4);
+        }
+    }
+    _lastHandTrackerFrameIndex = handTrackerFrameIndex;
+}
+
+
 void ViveControllerManager::pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
 
     if (!_system) {
@@ -297,6 +800,12 @@ void ViveControllerManager::pluginUpdate(float deltaTime, const controller::Inpu
         userInputMapper->registerDevice(_inputDevice);
         _registeredWithInputMapper = true;
     }
+
+    if (_viveProEye) {
+        updateEyeTracker(deltaTime, inputCalibrationData);
+    }
+
+    updateCameraHandTracker(deltaTime, inputCalibrationData);
 }
 
 void ViveControllerManager::loadSettings() {
@@ -830,9 +1339,7 @@ void ViveControllerManager::InputDevice::handleHmd(uint32_t deviceIndex, const c
 
 void ViveControllerManager::InputDevice::handleHandController(float deltaTime, uint32_t deviceIndex, const controller::InputCalibrationData& inputCalibrationData, bool isLeftHand) {
 
-    if (_system->IsTrackedDeviceConnected(deviceIndex) &&
+    if (isDeviceIndexActive(_system, deviceIndex) && _nextSimPoseData.vrPoses[deviceIndex].bPoseIsValid) {
-        _system->GetTrackedDeviceClass(deviceIndex) == vr::TrackedDeviceClass_Controller &&
-        _nextSimPoseData.vrPoses[deviceIndex].bPoseIsValid) {
 
         // process pose
         const mat4& mat = _nextSimPoseData.poses[deviceIndex];
@@ -1401,9 +1908,52 @@ controller::Input::NamedVector ViveControllerManager::InputDevice::getAvailableI
         makePair(LEFT_GRIP, "LeftGrip"),
         makePair(RIGHT_GRIP, "RightGrip"),
 
-        // 3d location of controller
+        // 3d location of left controller and fingers
         makePair(LEFT_HAND, "LeftHand"),
+        makePair(LEFT_HAND_THUMB1, "LeftHandThumb1"),
+        makePair(LEFT_HAND_THUMB2, "LeftHandThumb2"),
+        makePair(LEFT_HAND_THUMB3, "LeftHandThumb3"),
+        makePair(LEFT_HAND_THUMB4, "LeftHandThumb4"),
+        makePair(LEFT_HAND_INDEX1, "LeftHandIndex1"),
+        makePair(LEFT_HAND_INDEX2, "LeftHandIndex2"),
+        makePair(LEFT_HAND_INDEX3, "LeftHandIndex3"),
+        makePair(LEFT_HAND_INDEX4, "LeftHandIndex4"),
+        makePair(LEFT_HAND_MIDDLE1, "LeftHandMiddle1"),
+        makePair(LEFT_HAND_MIDDLE2, "LeftHandMiddle2"),
+        makePair(LEFT_HAND_MIDDLE3, "LeftHandMiddle3"),
+        makePair(LEFT_HAND_MIDDLE4, "LeftHandMiddle4"),
+        makePair(LEFT_HAND_RING1, "LeftHandRing1"),
+        makePair(LEFT_HAND_RING2, "LeftHandRing2"),
+        makePair(LEFT_HAND_RING3, "LeftHandRing3"),
+        makePair(LEFT_HAND_RING4, "LeftHandRing4"),
+        makePair(LEFT_HAND_PINKY1, "LeftHandPinky1"),
+        makePair(LEFT_HAND_PINKY2, "LeftHandPinky2"),
+        makePair(LEFT_HAND_PINKY3, "LeftHandPinky3"),
+        makePair(LEFT_HAND_PINKY4, "LeftHandPinky4"),
+
+        // 3d location of right controller and fingers
         makePair(RIGHT_HAND, "RightHand"),
+        makePair(RIGHT_HAND_THUMB1, "RightHandThumb1"),
+        makePair(RIGHT_HAND_THUMB2, "RightHandThumb2"),
+        makePair(RIGHT_HAND_THUMB3, "RightHandThumb3"),
+        makePair(RIGHT_HAND_THUMB4, "RightHandThumb4"),
+        makePair(RIGHT_HAND_INDEX1, "RightHandIndex1"),
+        makePair(RIGHT_HAND_INDEX2, "RightHandIndex2"),
+        makePair(RIGHT_HAND_INDEX3, "RightHandIndex3"),
+        makePair(RIGHT_HAND_INDEX4, "RightHandIndex4"),
+        makePair(RIGHT_HAND_MIDDLE1, "RightHandMiddle1"),
+        makePair(RIGHT_HAND_MIDDLE2, "RightHandMiddle2"),
+        makePair(RIGHT_HAND_MIDDLE3, "RightHandMiddle3"),
+        makePair(RIGHT_HAND_MIDDLE4, "RightHandMiddle4"),
+        makePair(RIGHT_HAND_RING1, "RightHandRing1"),
+        makePair(RIGHT_HAND_RING2, "RightHandRing2"),
+        makePair(RIGHT_HAND_RING3, "RightHandRing3"),
+        makePair(RIGHT_HAND_RING4, "RightHandRing4"),
+        makePair(RIGHT_HAND_PINKY1, "RightHandPinky1"),
+        makePair(RIGHT_HAND_PINKY2, "RightHandPinky2"),
+        makePair(RIGHT_HAND_PINKY3, "RightHandPinky3"),
+        makePair(RIGHT_HAND_PINKY4, "RightHandPinky4"),
+
         makePair(LEFT_FOOT, "LeftFoot"),
         makePair(RIGHT_FOOT, "RightFoot"),
         makePair(HIPS, "Hips"),
@@ -1411,6 +1961,10 @@ controller::Input::NamedVector ViveControllerManager::InputDevice::getAvailableI
         makePair(HEAD, "Head"),
         makePair(LEFT_ARM, "LeftArm"),
         makePair(RIGHT_ARM, "RightArm"),
+        makePair(LEFT_EYE, "LeftEye"),
+        makePair(RIGHT_EYE, "RightEye"),
+        makePair(EYEBLINK_L, "EyeBlink_L"),
+        makePair(EYEBLINK_R, "EyeBlink_R"),
 
         // 16 tracked poses
         makePair(TRACKED_OBJECT_00, "TrackedObject00"),

plugins/openvr/src/ViveControllerManager.h

@@ -31,6 +31,23 @@ namespace vr {
     class IVRSystem;
 }
 
+class ViveProEyeReadThread;
+
+class EyeDataBuffer {
+public:
+    int getEyeDataResult { 0 };
+    bool leftDirectionValid { false };
+    bool rightDirectionValid { false };
+    bool leftOpennessValid { false };
+    bool rightOpennessValid { false };
+    glm::vec3 leftEyeGaze;
+    glm::vec3 rightEyeGaze;
+    float leftEyeOpenness { 0.0f };
+    float rightEyeOpenness { 0.0f };
+};
+
+
+
 class ViveControllerManager : public InputPlugin {
     Q_OBJECT
 public:
@@ -49,12 +66,18 @@ public:
     bool isHeadController() const override { return true; }
     bool isHeadControllerMounted() const;
 
+    void enableGestureDetection();
+    void disableGestureDetection();
+
     bool activate() override;
     void deactivate() override;
 
     QString getDeviceName() { return QString::fromStdString(_inputDevice->_headsetName); }
 
     void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); }
+    void invalidateEyeInputs();
+    void updateEyeTracker(float deltaTime, const controller::InputCalibrationData& inputCalibrationData);
+    void updateCameraHandTracker(float deltaTime, const controller::InputCalibrationData& inputCalibrationData);
     void pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override;
 
     virtual void saveSettings() const override;
@@ -229,6 +252,23 @@ private:
     vr::IVRSystem* _system { nullptr };
     std::shared_ptr<InputDevice> _inputDevice { std::make_shared<InputDevice>(_system) };
 
+    bool _viveProEye { false };
+    std::shared_ptr<ViveProEyeReadThread> _viveProEyeReadThread;
+    EyeDataBuffer _prevEyeData;
+
+    bool _viveCameraHandTracker { false };
+    int _lastHandTrackerFrameIndex { -1 };
+
+    const static int NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES { 6 };
+    const static int NUMBER_OF_HAND_POINTS { 21 };
+    glm::vec3 _handPoints[NUMBER_OF_HAND_TRACKER_SMOOTHING_FRAMES][2][NUMBER_OF_HAND_POINTS]; // 2 for number of hands
+    glm::vec3 getRollingAverageHandPoint(int handIndex, int pointIndex) const;
+    controller::Pose trackedHandDataToPose(int hand, const glm::vec3& palmFacing,
+                                           int nearHandPositionIndex, int farHandPositionIndex);
+    void trackFinger(int hand, int jointIndex1, int jointIndex2, int jointIndex3, int jointIndex4,
+                     controller::StandardPoseChannel joint1, controller::StandardPoseChannel joint2,
+                     controller::StandardPoseChannel joint3, controller::StandardPoseChannel joint4);
+
     static const char* NAME;
 };
 

scripts/defaultScripts.js

@@ -34,7 +34,8 @@ var DEFAULT_SCRIPTS_COMBINED = [
     "system/miniTablet.js",
     "system/audioMuteOverlay.js",
     "system/inspect.js",
-    "system/keyboardShortcuts/keyboardShortcuts.js"
+    "system/keyboardShortcuts/keyboardShortcuts.js",
+    "system/hand-track-walk.js"
 ];
 var DEFAULT_SCRIPTS_SEPARATE = [
     "system/controllers/controllerScripts.js",

scripts/system/controllers/controllerDispatcher.js

@@ -35,7 +35,6 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
     var DEBUG = false;
     var SHOW_GRAB_SPHERE = false;
 
-
     if (typeof Test !== "undefined") {
         PROFILE = true;
     }
@@ -54,6 +53,8 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
         this.pointerManager = new PointerManager();
         this.grabSphereOverlays = [null, null];
         this.targetIDs = {};
+        this.debugPanelID = null;
+        this.debugLines = [];
 
         // a module can occupy one or more "activity" slots while it's running.  If all the required slots for a module are
         // not set to false (not in use), a module cannot start.  When a module is using a slot, that module's name
@@ -97,11 +98,15 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
         };
 
         this.runningPluginNames = {};
+
         this.leftTriggerValue = 0;
         this.leftTriggerClicked = 0;
+        this.leftTrackerClicked = false; // is leftTriggerClicked == 1 because a hand tracker set it?
+        this.leftSecondaryValue = 0;
+
         this.rightTriggerValue = 0;
         this.rightTriggerClicked = 0;
-        this.leftSecondaryValue = 0;
+        this.rightTrackerClicked = false; // is rightTriggerClicked == 1 because a hand tracker set it?
         this.rightSecondaryValue = 0;
 
         this.leftTriggerPress = function (value) {
@@ -162,6 +167,38 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
             }
         };
 
+        this.checkForHandTrackingClick = function() {
+
+            var pinchOnBelowDistance = 0.016;
+            var pinchOffAboveDistance = 0.035;
+
+            var leftIndexPose = Controller.getPoseValue(Controller.Standard.LeftHandIndex4);
+            var leftThumbPose = Controller.getPoseValue(Controller.Standard.LeftHandThumb4);
+            var leftThumbToIndexDistance = Vec3.distance(leftIndexPose.translation, leftThumbPose.translation);
+            if (leftIndexPose.valid && leftThumbPose.valid && leftThumbToIndexDistance < pinchOnBelowDistance) {
+                _this.leftTriggerClicked = 1;
+                _this.leftTriggerValue = 1;
+                _this.leftTrackerClicked = true;
+            } else if (_this.leftTrackerClicked && leftThumbToIndexDistance > pinchOffAboveDistance) {
+                _this.leftTriggerClicked = 0;
+                _this.leftTriggerValue = 0;
+                _this.leftTrackerClicked = false;
+            }
+
+            var rightIndexPose = Controller.getPoseValue(Controller.Standard.RightHandIndex4);
+            var rightThumbPose = Controller.getPoseValue(Controller.Standard.RightHandThumb4);
+            var rightThumbToIndexDistance = Vec3.distance(rightIndexPose.translation, rightThumbPose.translation);
+            if (rightIndexPose.valid && rightThumbPose.valid && rightThumbToIndexDistance < pinchOnBelowDistance) {
+                _this.rightTriggerClicked = 1;
+                _this.rightTriggerValue = 1;
+                _this.rightTrackerClicked = true;
+            } else if (_this.rightTrackerClicked && rightThumbToIndexDistance > pinchOffAboveDistance) {
+                _this.rightTriggerClicked = 0;
+                _this.rightTriggerValue = 0;
+                _this.rightTrackerClicked = false;
+            }
+        };
+
         this.update = function () {
             try {
                 _this.updateInternal();
@@ -171,6 +208,18 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
             Script.setTimeout(_this.update, BASIC_TIMER_INTERVAL_MS);
         };
 
+        this.addDebugLine = function(line) {
+            if (this.debugLines.length > 8) {
+                this.debugLines.shift();
+            }
+            this.debugLines.push(line);
+            var debugPanelText = "";
+            this.debugLines.forEach(function(debugLine) {
+                debugPanelText += debugLine + "\n";
+            });
+            Entities.editEntity(this.debugPanelID, { text: debugPanelText });
+        };
+
         this.updateInternal = function () {
             if (PROFILE) {
                 Script.beginProfileRange("dispatch.pre");
@@ -274,6 +323,7 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
                     }
 
                     var nearbyEntityIDs = Entities.findEntities(controllerPosition, findRadius);
+
                     for (var j = 0; j < nearbyEntityIDs.length; j++) {
                         var entityID = nearbyEntityIDs[j];
                         var props = Entities.getEntityProperties(entityID, DISPATCHER_PROPERTIES);
@@ -369,6 +419,9 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
                 }
             }
 
+            // check for hand-tracking "click"
+            _this.checkForHandTrackingClick();
+
             // bundle up all the data about the current situation
             var controllerData = {
                 triggerValues: [_this.leftTriggerValue, _this.rightTriggerValue],
@@ -406,7 +459,7 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
                         _this.markSlots(candidatePlugin, orderedPluginName);
                         _this.pointerManager.makePointerVisible(candidatePlugin.parameters.handLaser);
                         if (DEBUG) {
-                            print("controllerDispatcher running " + orderedPluginName);
+                            _this.addDebugLine("running " + orderedPluginName);
                         }
                     }
                     if (PROFILE) {
@@ -438,8 +491,8 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
 
                         if (DEBUG) {
                             if (JSON.stringify(_this.targetIDs[runningPluginName]) != JSON.stringify(runningness.targets)) {
-                                print("controllerDispatcher targetIDs[" + runningPluginName + "] = " +
+                                _this.addDebugLine("targetIDs[" + runningPluginName + "] = " +
-                                      JSON.stringify(runningness.targets));
+                                                  JSON.stringify(runningness.targets));
                             }
                         }
 
@@ -450,12 +503,12 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
                             delete _this.runningPluginNames[runningPluginName];
                             delete _this.targetIDs[runningPluginName];
                             if (DEBUG) {
-                                print("controllerDispatcher deleted targetIDs[" + runningPluginName + "]");
+                                _this.addDebugLine("deleted targetIDs[" + runningPluginName + "]");
                             }
                             _this.markSlots(plugin, false);
                             _this.pointerManager.makePointerInvisible(plugin.parameters.handLaser);
                             if (DEBUG) {
-                                print("controllerDispatcher stopping " + runningPluginName);
+                                _this.addDebugLine("stopping " + runningPluginName);
                             }
                         }
                         _this.pointerManager.lockPointerEnd(plugin.parameters.handLaser, runningness.laserLockInfo);
@@ -599,7 +652,33 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
             Overlays.mousePressOnOverlay.disconnect(mousePress);
             Entities.mousePressOnEntity.disconnect(mousePress);
             Messages.messageReceived.disconnect(controllerDispatcher.handleMessage);
+            if (_this.debugPanelID) {
+                Entities.deleteEntity(_this.debugPanelID);
+                _this.debugPanelID = null;
+            }
         };
+
+        if (DEBUG) {
+            this.debugPanelID = Entities.addEntity({
+                name: "controllerDispatcher debug panel",
+                type: "Text",
+                dimensions: { x: 1.0, y: 0.3, z: 0.01 },
+                parentID: MyAvatar.sessionUUID,
+                // parentJointIndex: MyAvatar.getJointIndex("_CAMERA_MATRIX"),
+                parentJointIndex: -1,
+                localPosition: { x: -0.25, y: 0.8, z: -1.2 },
+                textColor: { red: 255, green: 255, blue: 255},
+                backgroundColor: { red: 0, green: 0, blue: 0},
+                text: "",
+                lineHeight: 0.03,
+                leftMargin: 0.015,
+                topMargin: 0.01,
+                backgroundAlpha: 0.7,
+                textAlpha: 1.0,
+                unlit: true,
+                ignorePickIntersection: true
+            }, "local");
+        }
     }
 
     function mouseReleaseOnOverlay(overlayID, event) {
@@ -629,6 +708,8 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
     Messages.subscribe('Hifi-Hand-RayPick-Blacklist');
     Messages.messageReceived.connect(controllerDispatcher.handleMessage);
 
-    Script.scriptEnding.connect(controllerDispatcher.cleanup);
+    Script.scriptEnding.connect(function () {
+        controllerDispatcher.cleanup();
+    });
     Script.setTimeout(controllerDispatcher.update, BASIC_TIMER_INTERVAL_MS);
 }());

scripts/system/controllers/controllerModules/disableOtherModule.js

@@ -17,7 +17,7 @@ Script.include("/~/system/libraries/controllerDispatcherUtils.js");
         this.hand = hand;
         this.disableModules = false;
         this.parameters = makeDispatcherModuleParameters(
-            90,
+            95,
             this.hand === RIGHT_HAND ?
                 ["rightHand", "rightHandEquip", "rightHandTrigger"] :
                 ["leftHand", "leftHandEquip", "leftHandTrigger"],

scripts/system/controllers/controllerModules/farActionGrabEntity.js

@@ -14,7 +14,7 @@
    TRIGGER_OFF_VALUE, TRIGGER_ON_VALUE, ZERO_VEC, ensureDynamic,
    getControllerWorldLocation, projectOntoEntityXYPlane, ContextOverlay, HMD,
    Picks, makeLaserLockInfo, makeLaserParams, AddressManager, getEntityParents, Selection, DISPATCHER_HOVERING_LIST,
-   worldPositionToRegistrationFrameMatrix, DISPATCHER_PROPERTIES, Uuid, Picks
+   worldPositionToRegistrationFrameMatrix, DISPATCHER_PROPERTIES, Uuid, Picks, handsAreTracked, Messages
 */
 
 Script.include("/~/system/libraries/controllerDispatcherUtils.js");
@@ -374,6 +374,9 @@ Script.include("/~/system/libraries/controllers.js");
 
         this.isReady = function (controllerData) {
             if (HMD.active) {
+                if (handsAreTracked()) {
+                    return makeRunningValues(false, [], []);
+                }
                 if (this.notPointingAtEntity(controllerData)) {
                     return makeRunningValues(false, [], []);
                 }

scripts/system/controllers/controllerModules/farGrabEntity.js

@@ -12,7 +12,7 @@
    HAPTIC_PULSE_STRENGTH, HAPTIC_PULSE_DURATION, TRIGGER_OFF_VALUE, TRIGGER_ON_VALUE, ZERO_VEC,
    projectOntoEntityXYPlane, ContextOverlay, HMD, Picks, makeLaserLockInfo, makeLaserParams, AddressManager,
    getEntityParents, Selection, DISPATCHER_HOVERING_LIST, unhighlightTargetEntity, Messages, findGrabbableGroupParent,
-   worldPositionToRegistrationFrameMatrix, DISPATCHER_PROPERTIES
+   worldPositionToRegistrationFrameMatrix, DISPATCHER_PROPERTIES, handsAreTracked
 */
 
 Script.include("/~/system/libraries/controllerDispatcherUtils.js");
@@ -63,7 +63,6 @@ Script.include("/~/system/libraries/controllers.js");
         this.endedGrab = 0;
         this.MIN_HAPTIC_PULSE_INTERVAL = 500; // ms
         this.disabled = false;
-        var _this = this;
         this.initialControllerRotation = Quat.IDENTITY;
         this.currentControllerRotation = Quat.IDENTITY;
         this.manipulating = false;
@@ -99,7 +98,7 @@ Script.include("/~/system/libraries/controllers.js");
 
         this.getOffhand = function () {
             return (this.hand === RIGHT_HAND ? LEFT_HAND : RIGHT_HAND);
-        }
+        };
 
         // Activation criteria for rotating a fargrabbed entity. If we're changing the mapping, this is where to do it.
         this.shouldManipulateTarget = function (controllerData) {
@@ -406,6 +405,9 @@ Script.include("/~/system/libraries/controllers.js");
 
         this.isReady = function (controllerData) {
             if (HMD.active) {
+                if (handsAreTracked()) {
+                    return makeRunningValues(false, [], []);
+                }
                 if (this.notPointingAtEntity(controllerData)) {
                     return makeRunningValues(false, [], []);
                 }

scripts/system/controllers/controllerModules/farTrigger.js

@@ -8,7 +8,7 @@
 
 /* global Script, RIGHT_HAND, LEFT_HAND, MyAvatar,
    makeRunningValues, Entities, enableDispatcherModule, disableDispatcherModule, makeDispatcherModuleParameters,
-   getGrabbableData, makeLaserParams, DISPATCHER_PROPERTIES
+   getGrabbableData, makeLaserParams, DISPATCHER_PROPERTIES, RayPick, handsAreTracked
 */
 
 Script.include("/~/system/libraries/controllerDispatcherUtils.js");
@@ -63,6 +63,9 @@ Script.include("/~/system/libraries/controllers.js");
 
         this.isReady = function (controllerData) {
             this.targetEntityID = null;
+            if (handsAreTracked()) {
+                return makeRunningValues(false, [], []);
+            }
             if (controllerData.triggerClicks[this.hand] === 0) {
                 return makeRunningValues(false, [], []);
             }

scripts/system/controllers/controllerModules/nearGrabEntity.js

@@ -151,7 +151,7 @@ Script.include("/~/system/libraries/controllers.js");
         this.run = function (controllerData, deltaTime) {
 
             if (this.grabbing) {
-                if (controllerData.triggerClicks[this.hand] < TRIGGER_OFF_VALUE &&
+                if (!controllerData.triggerClicks[this.hand] &&
                     controllerData.secondaryValues[this.hand] < TRIGGER_OFF_VALUE) {
                     this.endNearGrabEntity();
                     return makeRunningValues(false, [], []);

scripts/system/controllers/controllerModules/stylusInput.js

@@ -7,7 +7,7 @@
 
 /* global Script, MyAvatar, Controller, Uuid, RIGHT_HAND, LEFT_HAND, enableDispatcherModule, disableDispatcherModule,
    makeRunningValues, Vec3, makeDispatcherModuleParameters, Overlays, HMD, Settings, getEnabledModuleByName, Pointers,
-   Picks, PickType
+   Picks, PickType, Keyboard
 */
 
 Script.include("/~/system/libraries/controllerDispatcherUtils.js");
@@ -52,22 +52,21 @@ Script.include("/~/system/libraries/controllers.js");
         this.disable = false;
 
         this.otherModuleNeedsToRun = function(controllerData) {
-            var grabOverlayModuleName = this.hand === RIGHT_HAND ? "RightNearParentingGrabOverlay" : "LeftNearParentingGrabOverlay";
+            var grabEntityModuleName = this.hand === RIGHT_HAND ? "RightNearGrabEntity" : "LeftNearGrabEntity";
-            var grabOverlayModule = getEnabledModuleByName(grabOverlayModuleName);
-            var grabEntityModuleName = this.hand === RIGHT_HAND ? "RightNearParentingGrabEntity" : "LeftNearParentingGrabEntity";
             var grabEntityModule = getEnabledModuleByName(grabEntityModuleName);
-            var grabOverlayModuleReady = grabOverlayModule ? grabOverlayModule.isReady(controllerData) : makeRunningValues(false, [], []);
             var grabEntityModuleReady = grabEntityModule ? grabEntityModule.isReady(controllerData) : makeRunningValues(false, [], []);
-            var farGrabModuleName = this.hand === RIGHT_HAND ? "RightFarActionGrabEntity" : "LeftFarActionGrabEntity";
+
+            var farGrabModuleName = this.hand === RIGHT_HAND ? "RightFarGrabEntity" : "LeftFarGrabEntity";
             var farGrabModule = getEnabledModuleByName(farGrabModuleName);
             var farGrabModuleReady = farGrabModule ? farGrabModule.isReady(controllerData) : makeRunningValues(false, [], []);
+
             var nearTabletHighlightModuleName =
                 this.hand === RIGHT_HAND ? "RightNearTabletHighlight" : "LeftNearTabletHighlight";
             var nearTabletHighlightModule = getEnabledModuleByName(nearTabletHighlightModuleName);
-            var nearTabletHighlightModuleReady = nearTabletHighlightModule
+            var nearTabletHighlightModuleReady = nearTabletHighlightModule ?
-                ? nearTabletHighlightModule.isReady(controllerData) : makeRunningValues(false, [], []);
+                nearTabletHighlightModule.isReady(controllerData) : makeRunningValues(false, [], []);
-            return grabOverlayModuleReady.active || farGrabModuleReady.active || grabEntityModuleReady.active
+
-                || nearTabletHighlightModuleReady.active;
+            return farGrabModuleReady.active || grabEntityModuleReady.active;
         };
 
         this.overlayLaserActive = function(controllerData) {
@@ -121,14 +120,15 @@ Script.include("/~/system/libraries/controllers.js");
             }
 
             // Add the mini tablet.
-            if (HMD.miniTabletScreenID && Overlays.getProperty(HMD.miniTabletScreenID, "visible")) {
+            if (HMD.miniTabletScreenID && Overlays.getProperty(HMD.miniTabletScreenID, "visible") &&
+                this.hand != HMD.miniTabletHand) {
                 stylusTarget = getOverlayDistance(controllerPosition, HMD.miniTabletScreenID);
                 if (stylusTarget) {
                     stylusTargets.push(stylusTarget);
                 }
             }
 
-            const WEB_DISPLAY_STYLUS_DISTANCE = (Keyboard.raised && Keyboard.preferMalletsOverLasers) ? 0.2 : 0.5;
+            var WEB_DISPLAY_STYLUS_DISTANCE = (Keyboard.raised && Keyboard.preferMalletsOverLasers) ? 0.2 : 0.5;
             var nearStylusTarget = isNearStylusTarget(stylusTargets, WEB_DISPLAY_STYLUS_DISTANCE * sensorScaleFactor);
 
             if (nearStylusTarget.length !== 0) {

scripts/system/controllers/controllerModules/trackedHandTablet.js

@@ -0,0 +1,136 @@
+"use strict";
+
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+
+/* global Script, makeRunningValues, enableDispatcherModule, disableDispatcherModule,
+   makeDispatcherModuleParameters, handsAreTracked, Controller, Vec3, Tablet, HMD, MyAvatar
+*/
+
+Script.include("/~/system/libraries/controllerDispatcherUtils.js");
+Script.include("/~/system/libraries/controllers.js");
+
+(function() {
+
+    function TrackedHandTablet() {
+        this.mappingName = 'hand-track-tablet-' + Math.random();
+        this.inputMapping = Controller.newMapping(this.mappingName);
+        this.leftIndexPos = null;
+        this.leftThumbPos = null;
+        this.rightIndexPos = null;
+        this.rightThumbPos = null;
+        this.touchOnBelowDistance = 0.016;
+        this.touchOffAboveDistance = 0.045;
+
+        this.gestureCompleted = false;
+        this.previousGestureCompleted = false;
+
+        this.parameters = makeDispatcherModuleParameters(
+            70,
+            ["rightHand", "leftHand"],
+            [],
+            100);
+
+        this.checkForGesture = function () {
+            if (this.leftThumbPos && this.leftIndexPos && this.rightThumbPos && this.rightIndexPos) {
+                var leftTipDistance = Vec3.distance(this.leftThumbPos, this.leftIndexPos);
+                var rightTipDistance = Vec3.distance(this.rightThumbPos, this.rightIndexPos);
+                if (leftTipDistance < this.touchOnBelowDistance && rightTipDistance < this.touchOnBelowDistance) {
+                    this.gestureCompleted = true;
+                } else if (leftTipDistance > this.touchOffAboveDistance || rightTipDistance > this.touchOffAboveDistance) {
+                    this.gestureCompleted = false;
+                } // else don't change gestureCompleted
+            } else {
+                this.gestureCompleted = false;
+            }
+
+            if (this.gestureCompleted && !this.previousGestureCompleted) {
+                var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
+                if (HMD.showTablet) {
+                    HMD.closeTablet(false);
+                } else if (!HMD.showTablet && !tablet.toolbarMode && !MyAvatar.isAway) {
+                    tablet.gotoHomeScreen();
+                    HMD.openTablet(false);
+                }
+            }
+
+            this.previousGestureCompleted = this.gestureCompleted;
+        };
+
+        this.leftIndexChanged = function (pose) {
+            if (pose.valid) {
+                this.leftIndexPos = pose.translation;
+            } else {
+                this.leftIndexPos = null;
+            }
+            this.checkForGesture();
+        };
+
+        this.leftThumbChanged = function (pose) {
+            if (pose.valid) {
+                this.leftThumbPos = pose.translation;
+            } else {
+                this.leftThumbPos = null;
+            }
+            this.checkForGesture();
+        };
+
+        this.rightIndexChanged = function (pose) {
+            if (pose.valid) {
+                this.rightIndexPos = pose.translation;
+            } else {
+                this.rightIndexPos = null;
+            }
+            this.checkForGesture();
+        };
+
+        this.rightThumbChanged = function (pose) {
+            if (pose.valid) {
+                this.rightThumbPos = pose.translation;
+            } else {
+                this.rightThumbPos = null;
+            }
+            this.checkForGesture();
+        };
+
+        this.isReady = function (controllerData) {
+            return makeRunningValues(handsAreTracked() && this.gestureCompleted, [], []);
+        };
+
+        this.run = function (controllerData) {
+            return this.isReady(controllerData);
+        };
+
+        this.setup = function () {
+            var _this = this;
+            this.inputMapping.from(Controller.Standard.LeftHandIndex4).peek().to(function (pose) {
+                _this.leftIndexChanged(pose);
+            });
+            this.inputMapping.from(Controller.Standard.LeftHandThumb4).peek().to(function (pose) {
+                _this.leftThumbChanged(pose);
+            });
+            this.inputMapping.from(Controller.Standard.RightHandIndex4).peek().to(function (pose) {
+                _this.rightIndexChanged(pose);
+            });
+            this.inputMapping.from(Controller.Standard.RightHandThumb4).peek().to(function (pose) {
+                _this.rightThumbChanged(pose);
+            });
+
+            Controller.enableMapping(this.mappingName);
+        };
+
+        this.cleanUp = function () {
+            this.inputMapping.disable();
+        };
+    }
+
+    var trackedHandWalk = new TrackedHandTablet();
+    trackedHandWalk.setup();
+    enableDispatcherModule("TrackedHandTablet", trackedHandWalk);
+
+    function cleanup() {
+        trackedHandWalk.cleanUp();
+        disableDispatcherModule("TrackedHandTablet");
+    }
+    Script.scriptEnding.connect(cleanup);
+}());

scripts/system/controllers/controllerModules/trackedHandWalk.js

@@ -0,0 +1,172 @@
+"use strict";
+
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+
+/* global Script, makeRunningValues, enableDispatcherModule, disableDispatcherModule,
+   makeDispatcherModuleParameters, handsAreTracked, Controller, Vec3
+*/
+
+Script.include("/~/system/libraries/controllerDispatcherUtils.js");
+Script.include("/~/system/libraries/controllers.js");
+
+(function() {
+
+    function TrackedHandWalk() {
+        this.gestureMappingName = 'hand-track-walk-gesture-' + Math.random();
+        this.inputGestureMapping = Controller.newMapping(this.gestureMappingName);
+        this.mappingName = 'hand-track-walk-' + Math.random();
+        this.inputMapping = Controller.newMapping(this.mappingName);
+        this.leftIndexPos = null;
+        this.leftThumbPos = null;
+        this.rightIndexPos = null;
+        this.rightThumbPos = null;
+        this.touchOnBelowDistance = 0.016;
+        this.touchOffAboveDistance = 0.045;
+        this.walkingForward = false;
+        this.walkingBackward = false;
+
+        this.mappingEnabled = false;
+
+        this.parameters = makeDispatcherModuleParameters(
+            80,
+            ["rightHand", "leftHand"],
+            [],
+            100);
+
+        this.getControlPoint = function () {
+            return Vec3.multiply(Vec3.sum(this.leftIndexPos, this.rightIndexPos), 0.5);
+        };
+
+        this.updateWalking = function () {
+            if (this.leftIndexPos && this.rightIndexPos) {
+                var indexTipDistance = Vec3.distance(this.leftIndexPos, this.rightIndexPos);
+                if (indexTipDistance < this.touchOnBelowDistance) {
+                    this.walkingForward = true;
+                    this.controlPoint = this.getControlPoint();
+                } else if (this.walkingForward && indexTipDistance > this.touchOffAboveDistance) {
+                    this.walkingForward = false;
+                } // else don't change walkingForward
+            }
+
+            if (this.leftThumbPos && this.rightThumbPos) {
+                var thumbTipDistance = Vec3.distance(this.leftThumbPos, this.rightThumbPos);
+                if (thumbTipDistance < this.touchOnBelowDistance) {
+                    this.walkingBackward = true;
+                    this.controlPoint = this.getControlPoint();
+                } else if (this.walkingBackward && thumbTipDistance > this.touchOffAboveDistance) {
+                    this.walkingBackward = false;
+                } // else don't change this.walkingBackward
+            }
+
+            if ((this.walkingForward || this.walkingBackward) && !this.mappingEnabled) {
+                Controller.enableMapping(this.mappingName);
+                this.mappingEnabled = true;
+            } else if (!(this.walkingForward || this.walkingBackward) && this.mappingEnabled) {
+                this.inputMapping.disable();
+                this.mappingEnabled = false;
+            } // else don't change mappingEnabled
+        };
+
+        this.leftIndexChanged = function (pose) {
+            if (pose.valid) {
+                this.leftIndexPos = pose.translation;
+            } else {
+                this.leftIndexPos = null;
+            }
+            this.updateWalking();
+        };
+
+        this.leftThumbChanged = function (pose) {
+            if (pose.valid) {
+                this.leftThumbPos = pose.translation;
+            } else {
+                this.leftThumbPos = null;
+            }
+            this.updateWalking();
+        };
+
+        this.rightIndexChanged = function (pose) {
+            if (pose.valid) {
+                this.rightIndexPos = pose.translation;
+            } else {
+                this.rightIndexPos = null;
+            }
+            this.updateWalking();
+        };
+
+        this.rightThumbChanged = function (pose) {
+            if (pose.valid) {
+                this.rightThumbPos = pose.translation;
+            } else {
+                this.rightThumbPos = null;
+            }
+            this.updateWalking();
+        };
+
+        this.isReady = function (controllerData) {
+            return makeRunningValues(handsAreTracked() && (this.walkingForward || this.walkingBackward), [], []);
+        };
+
+        this.run = function (controllerData) {
+            return this.isReady(controllerData);
+        };
+
+        this.setup = function () {
+            var _this = this;
+            this.inputGestureMapping.from(Controller.Standard.LeftHandIndex4).peek().to(function (pose) {
+                _this.leftIndexChanged(pose);
+            });
+            this.inputGestureMapping.from(Controller.Standard.LeftHandThumb4).peek().to(function (pose) {
+                _this.leftThumbChanged(pose);
+            });
+            this.inputGestureMapping.from(Controller.Standard.RightHandIndex4).peek().to(function (pose) {
+                _this.rightIndexChanged(pose);
+            });
+            this.inputGestureMapping.from(Controller.Standard.RightHandThumb4).peek().to(function (pose) {
+                _this.rightThumbChanged(pose);
+            });
+
+            this.inputMapping.from(function() {
+                if (_this.walkingForward) {
+                    // var currentPoint = _this.getControlPoint();
+                    // return currentPoint.z - _this.controlPoint.z;
+                    return -0.5;
+                } else if (_this.walkingBackward) {
+                    // var currentPoint = _this.getControlPoint();
+                    // return currentPoint.z - _this.controlPoint.z;
+                    return 0.5;
+                } else {
+                    // return Controller.getActionValue(Controller.Standard.TranslateZ);
+                    return 0.0;
+                }
+            }).to(Controller.Actions.TranslateZ);
+
+            // this.inputMapping.from(function() {
+            //     if (_this.walkingForward) {
+            //         var currentPoint = _this.getControlPoint();
+            //         return currentPoint.x - _this.controlPoint.x;
+            //     } else {
+            //         return Controller.getActionValue(Controller.Standard.Yaw);
+            //     }
+            // }).to(Controller.Actions.Yaw);
+
+            Controller.enableMapping(this.gestureMappingName);
+        };
+
+        this.cleanUp = function () {
+            this.inputGestureMapping.disable();
+            this.inputMapping.disable();
+        };
+    }
+
+    var trackedHandWalk = new TrackedHandWalk();
+    trackedHandWalk.setup();
+    enableDispatcherModule("TrackedHandWalk", trackedHandWalk);
+
+    function cleanup() {
+        trackedHandWalk.cleanUp();
+        disableDispatcherModule("TrackedHandWalk");
+    }
+    Script.scriptEnding.connect(cleanup);
+}());

scripts/system/controllers/controllerScripts.js

@@ -34,7 +34,9 @@ var CONTOLLER_SCRIPTS = [
     "controllerModules/nearTabletHighlight.js",
     "controllerModules/nearGrabEntity.js",
     "controllerModules/farGrabEntity.js",
-    "controllerModules/pushToTalk.js"
+    "controllerModules/pushToTalk.js",
+    "controllerModules/trackedHandWalk.js",
+    "controllerModules/trackedHandTablet.js"
 ];
 
 var DEBUG_MENU_ITEM = "Debug defaultScripts.js";

scripts/system/html/css/miniHandTablet.css

@@ -0,0 +1,56 @@
+/*
+miniTablet.css
+
+Copyright 2019 High Fidelity, Inc.
+
+Distributed under the Apache License, Version 2.0.
+See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+*/
+
+* {
+    box-sizing: border-box;
+    padding: 0;
+    margin: 0;
+    user-select: none;
+}
+
+html {
+    background-color: #404040;
+}
+
+body {
+    height: 100%;
+}
+
+section {
+    background-color: #404040;
+    position: relative;
+    padding: 32px 0px;
+}
+
+.button {
+    text-align: center;
+}
+
+img {
+    width: 149px;
+    height: 149px;
+}
+
+#expand {
+    width: 149px;
+    height: 149px;
+    background-size: 100% 100%;
+    background-image: url("./img/mt-expand-normal.svg");
+}
+
+#expand:hover {
+    background-image: url("./img/mt-expand-hover.svg");
+}
+
+#expand:hover.unhover {
+    background-image: url("./img/mt-expand-normal.svg");
+}
+
+#expand img {
+}

scripts/system/html/js/miniTablet.js

@@ -35,7 +35,9 @@
 
     function setUnhover() {
         if (!isUnhover) {
-            gotoButton.classList.add("unhover");
+            if (gotoButton) {
+                gotoButton.classList.add("unhover");
+            }
             expandButton.classList.add("unhover");
             isUnhover = true;
         }
@@ -43,7 +45,9 @@
 
     function clearUnhover() {
         if (isUnhover) {
-            gotoButton.classList.remove("unhover");
+            if (gotoButton) {
+                gotoButton.classList.remove("unhover");
+            }
             expandButton.classList.remove("unhover");
             isUnhover = false;
         }
@@ -62,10 +66,14 @@
 
         switch (message.type) {
             case MUTE_MESSAGE:
-                muteImage.src = message.icon;
+                if (muteImage) {
+                    muteImage.src = message.icon;
+                }
                 break;
             case GOTO_MESSAGE:
-                gotoImage.src = message.icon;
+                if (gotoImage) {
+                    gotoImage.src = message.icon;
+                }
                 break;
         }
     }
@@ -130,9 +138,7 @@
 
     function onLoad() {
         muteButton = document.getElementById("mute");
-        muteImage = document.getElementById("mute-img");
         gotoButton = document.getElementById("goto");
-        gotoImage = document.getElementById("goto-img");
         expandButton = document.getElementById("expand");
 
         connectEventBridge();
@@ -140,11 +146,19 @@
         document.body.addEventListener("mouseenter", onBodyHover, false);
         document.body.addEventListener("mouseleave", onBodyUnhover, false);
 
-        muteButton.addEventListener("mouseenter", onButtonHover, false);
+        if (muteButton) {
-        gotoButton.addEventListener("mouseenter", onButtonHover, false);
+            muteImage = document.getElementById("mute-img");
+            muteButton.addEventListener("mouseenter", onButtonHover, false);
+            muteButton.addEventListener("click", onMuteButtonClick, true);
+        }
+
+        if (gotoButton) {
+            gotoImage = document.getElementById("goto-img");
+            gotoButton.addEventListener("mouseenter", onButtonHover, false);
+            gotoButton.addEventListener("click", onGotoButtonClick, true);
+        }
+
         expandButton.addEventListener("mouseenter", onButtonHover, false);
-        muteButton.addEventListener("click", onMuteButtonClick, true);
-        gotoButton.addEventListener("click", onGotoButtonClick, true);
         expandButton.addEventListener("click", onExpandButtonClick, true);
 
         document.body.onunload = function () {

scripts/system/html/miniHandsTablet.html

@@ -0,0 +1,26 @@
+<!--
+miniTablet.html
+
+Created by David Rowe on 20 Aug 2018.
+Copyright 2018 High Fidelity, Inc.
+
+Distributed under the Apache License, Version 2.0.
+See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+-->
+
+<!DOCTYPE html>
+
+<html>
+<head>
+    <meta charset="utf-8" />
+    <link rel="stylesheet" type="text/css" href="css/miniHandTablet.css" />
+</head>
+<body>
+    <section>
+        <div id="expand" class="button">
+            <img src="./img/expand.svg" />
+        </div>
+    </section>
+    <script src="js/miniTablet.js"></script>
+</body>
+</html>

scripts/system/libraries/WebTablet.js

@@ -164,6 +164,7 @@ WebTablet = function (url, width, dpi, hand, location, visible) {
         parentID: this.tabletEntityID,
         parentJointIndex: -1,
         showKeyboardFocusHighlight: false,
+        grabbable: false,
         visible: visible
     });
 

scripts/system/libraries/controllerDispatcherUtils.js

@@ -6,7 +6,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 
 /* global module, HMD, MyAvatar, controllerDispatcherPlugins:true, Quat, Vec3, Overlays, Xform, Mat4,
-   Selection, Uuid,
+   Selection, Uuid, Controller,
    MSECS_PER_SEC:true , LEFT_HAND:true, RIGHT_HAND:true, FORBIDDEN_GRAB_TYPES:true,
    HAPTIC_PULSE_STRENGTH:true, HAPTIC_PULSE_DURATION:true, ZERO_VEC:true, ONE_VEC:true,
    DEFAULT_REGISTRATION_POINT:true, INCHES_TO_METERS:true,
@@ -56,6 +56,7 @@
    TEAR_AWAY_DISTANCE:true,
    TEAR_AWAY_COUNT:true,
    TEAR_AWAY_CHECK_TIME:true,
+   TELEPORT_DEADZONE: true,
    NEAR_GRAB_DISTANCE: true,
    distanceBetweenPointAndEntityBoundingBox:true,
    entityIsEquipped:true,
@@ -63,7 +64,8 @@
    clearHighlightedEntities:true,
    unhighlightTargetEntity:true,
    distanceBetweenEntityLocalPositionAndBoundingBox: true,
-   worldPositionToRegistrationFrameMatrix: true
+   worldPositionToRegistrationFrameMatrix: true,
+   handsAreTracked: true
 */
 
 MSECS_PER_SEC = 1000.0;
@@ -600,6 +602,10 @@ worldPositionToRegistrationFrameMatrix = function(wptrProps, pos) {
     return offsetMat;
 };
 
+handsAreTracked = function () {
+    return Controller.getPoseValue(Controller.Standard.LeftHandIndex3).valid ||
+        Controller.getPoseValue(Controller.Standard.RightHandIndex3).valid;
+};
 
 if (typeof module !== 'undefined') {
     module.exports = {
@@ -624,6 +630,7 @@ if (typeof module !== 'undefined') {
         TRIGGER_OFF_VALUE: TRIGGER_OFF_VALUE,
         TRIGGER_ON_VALUE: TRIGGER_ON_VALUE,
         DISPATCHER_HOVERING_LIST: DISPATCHER_HOVERING_LIST,
-        worldPositionToRegistrationFrameMatrix: worldPositionToRegistrationFrameMatrix
+        worldPositionToRegistrationFrameMatrix: worldPositionToRegistrationFrameMatrix,
+        handsAreTracked: handsAreTracked
     };
 }

scripts/system/miniTablet.js

@@ -8,7 +8,8 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
-/* global getTabletWidthFromSettings, TRIGGER_OFF_VALUE */
+/* global getTabletWidthFromSettings, handsAreTracked, TRIGGER_OFF_VALUE, Controller, Script, Camera, Tablet, MyAvatar,
+   Quat, SoundCache, HMD, Overlays, Vec3, Uuid, Messages */
 
 (function () {
 
@@ -80,7 +81,6 @@
         return hand === LEFT_HAND ? RIGHT_HAND : LEFT_HAND;
     }
 
-
     UI = function () {
 
         if (!(this instanceof UI)) {
@@ -114,6 +114,7 @@
             uiHand = LEFT_HAND,
             miniUIOverlay = null,
             MINI_UI_HTML = Script.resolvePath("./html/miniTablet.html"),
+            MINI_HAND_UI_HTML = Script.resolvePath("./html/miniHandsTablet.html"),
             MINI_UI_DIMENSIONS = { x: 0.059, y: 0.0865, z: 0.001 },
             MINI_UI_WIDTH_PIXELS = 150,
             METERS_TO_INCHES = 39.3701,
@@ -291,6 +292,7 @@
                 visible: true
             });
             Overlays.editOverlay(miniUIOverlay, {
+                url: handsAreTracked() ? MINI_HAND_UI_HTML : MINI_UI_HTML,
                 localPosition: Vec3.multiply(MyAvatar.sensorToWorldScale, MINI_UI_LOCAL_POSITION),
                 localRotation: MINI_UI_LOCAL_ROTATION,
                 dimensions: Vec3.multiply(initialScale, MINI_UI_DIMENSIONS),
@@ -353,8 +355,8 @@
                 localRotation,
                 localPosition;
 
-            tabletScaleFactor = MyAvatar.sensorToWorldScale
+            tabletScaleFactor = MyAvatar.sensorToWorldScale *
-                * (1 + scaleFactor * (miniTargetWidth - miniInitialWidth) / miniInitialWidth);
+                (1 + scaleFactor * (miniTargetWidth - miniInitialWidth) / miniInitialWidth);
             dimensions = Vec3.multiply(tabletScaleFactor, MINI_DIMENSIONS);
             localRotation = Quat.mix(miniExpandLocalRotation, miniTargetLocalRotation, scaleFactor);
             localPosition =
@@ -469,11 +471,11 @@
                 solid: true,
                 grabbable: true,
                 showKeyboardFocusHighlight: false,
-                drawInFront: true,
+                drawInFront: false,
                 visible: false
             });
             miniUIOverlay = Overlays.addOverlay("web3d", {
-                url: MINI_UI_HTML,
+                url: handsAreTracked() ? MINI_HAND_UI_HTML : MINI_UI_HTML,
                 parentID: miniOverlay,
                 localPosition: Vec3.multiply(MyAvatar.sensorToWorldScale, MINI_UI_LOCAL_POSITION),
                 localRotation: MINI_UI_LOCAL_ROTATION,
@@ -482,7 +484,7 @@
                 alpha: 0, // Hide overlay while its content is being created.
                 grabbable: false,
                 showKeyboardFocusHighlight: false,
-                drawInFront: true,
+                drawInFront: false,
                 visible: false
             });
 
@@ -642,8 +644,8 @@
             // is grabbing something) or the other hand's trigger is pressed unless it is pointing at the mini tablet. Allow 
             // the triggers to be pressed briefly to allow for the grabbing process.
             if (show) {
-                isLeftTriggerOff = Controller.getValue(Controller.Standard.LT) < TRIGGER_OFF_VALUE
+                isLeftTriggerOff = Controller.getValue(Controller.Standard.LT) < TRIGGER_OFF_VALUE &&
-                    && Controller.getValue(Controller.Standard.LeftGrip) < TRIGGER_OFF_VALUE;
+                    Controller.getValue(Controller.Standard.LeftGrip) < TRIGGER_OFF_VALUE;
                 if (!isLeftTriggerOff) {
                     if (leftTriggerOn === 0) {
                         leftTriggerOn = Date.now();
@@ -653,8 +655,8 @@
                 } else {
                     leftTriggerOn = 0;
                 }
-                isRightTriggerOff = Controller.getValue(Controller.Standard.RT) < TRIGGER_OFF_VALUE
+                isRightTriggerOff = Controller.getValue(Controller.Standard.RT) < TRIGGER_OFF_VALUE &&
-                    && Controller.getValue(Controller.Standard.RightGrip) < TRIGGER_OFF_VALUE;
+                    Controller.getValue(Controller.Standard.RightGrip) < TRIGGER_OFF_VALUE;
                 if (!isRightTriggerOff) {
                     if (rightTriggerOn === 0) {
                         rightTriggerOn = Date.now();
@@ -665,8 +667,8 @@
                     rightTriggerOn = 0;
                 }
 
-                show = (hand === LEFT_HAND ? wasLeftTriggerOff : wasRightTriggerOff)
+                show = (hand === LEFT_HAND ? wasLeftTriggerOff : wasRightTriggerOff) &&
-                    && ((hand === LEFT_HAND ? wasRightTriggerOff : wasLeftTriggerOff) || ui.isLaserPointingAt());
+                    ((hand === LEFT_HAND ? wasRightTriggerOff : wasLeftTriggerOff) || ui.isLaserPointingAt());
             }
 
             // Should show mini tablet if it would be oriented toward the camera.
@@ -691,10 +693,10 @@
                 normalDot = Vec3.dot(normalHandVector, miniToCameraDirection);
                 medialAngle = Math.atan2(medialDot, normalDot);
                 lateralAngle = Math.atan2(lateralDot, normalDot);
-                show = -MAX_MEDIAL_WRIST_CAMERA_ANGLE_RAD <= medialAngle
+                show = -MAX_MEDIAL_WRIST_CAMERA_ANGLE_RAD <= medialAngle &&
-                    && medialAngle <= MAX_MEDIAL_FINGER_CAMERA_ANGLE_RAD
+                    medialAngle <= MAX_MEDIAL_FINGER_CAMERA_ANGLE_RAD &&
-                    && -MAX_LATERAL_THUMB_CAMERA_ANGLE_RAD <= lateralAngle
+                    -MAX_LATERAL_THUMB_CAMERA_ANGLE_RAD <= lateralAngle &&
-                    && lateralAngle <= MAX_LATERAL_PINKY_CAMERA_ANGLE_RAD;
+                    lateralAngle <= MAX_LATERAL_PINKY_CAMERA_ANGLE_RAD;
 
                 // Camera looking at mini tablet?
                 cameraToMini = -Vec3.dot(miniToCameraDirection, Quat.getForward(Camera.orientation));
@@ -972,8 +974,8 @@
 
         function setState(state, data) {
             if (state !== miniState) {
-                debug("State transition from " + STATE_STRINGS[miniState] + " to " + STATE_STRINGS[state]
+                debug("State transition from " + STATE_STRINGS[miniState] + " to " + STATE_STRINGS[state] +
-                    + ( data ? " " + JSON.stringify(data) : ""));
+                      ( data ? " " + JSON.stringify(data) : ""));
                 if (STATE_MACHINE[STATE_STRINGS[miniState]].exit) {
                     STATE_MACHINE[STATE_STRINGS[miniState]].exit(data);
                 }
@@ -1061,8 +1063,8 @@
                 return;
         }
 
-        if (miniState.getState() === miniState.MINI_DISABLED
+        if (miniState.getState() === miniState.MINI_DISABLED ||
-                || (message.grabbedEntity !== HMD.tabletID && message.grabbedEntity !== ui.getMiniTabletID())) {
+            (message.grabbedEntity !== HMD.tabletID && message.grabbedEntity !== ui.getMiniTabletID())) {
             return;
         }