Merge pull request #9186 from ZappoMan/kinectPlugin

Kinect Plugin

cmake/macros/TargetKinect.cmake

@@ -0,0 +1,17 @@
+#
+#  Created by Brad Hefta-Gaub on 2016/12/7
+#  Copyright 2016 High Fidelity, Inc.
+#
+#  Distributed under the Apache License, Version 2.0.
+#  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+#
+macro(TARGET_KINECT)
+    # Kinect SDK data reader is only available on these platforms
+    if (WIN32)
+        #add_dependency_external_projects(kinect)
+        find_package(Kinect REQUIRED)
+        target_include_directories(${TARGET_NAME} PRIVATE ${KINECT_INCLUDE_DIRS})
+        target_link_libraries(${TARGET_NAME} ${KINECT_LIBRARIES})
+        add_definitions(-DHAVE_KINECT)
+    endif(WIN32)
+endmacro()

cmake/modules/FindKinect.cmake

@@ -0,0 +1,59 @@
+#
+#  FindKinect.cmake
+#
+#  Try to find the Perception Kinect SDK
+#
+#  You must provide a KINECT_ROOT_DIR which contains lib and include directories
+#
+#  Once done this will define
+#
+#  KINECT_FOUND - system found Kinect SDK
+#  KINECT_INCLUDE_DIRS - the Kinect SDK include directory
+#  KINECT_LIBRARIES - Link this to use Kinect
+#
+#  Created by Brad Hefta-Gaub on 2016/12/7
+#  Copyright 2016 High Fidelity, Inc.
+#
+#  Distributed under the Apache License, Version 2.0.
+#  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+#
+
+include("${MACRO_DIR}/HifiLibrarySearchHints.cmake")
+hifi_library_search_hints("kinect")
+
+find_path(KINECT_INCLUDE_DIRS Kinect.h PATH_SUFFIXES inc HINTS $ENV{KINECT_ROOT_DIR})
+
+if (WIN32)
+
+  if ("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
+      set(ARCH_DIR "x64")
+  else()
+      set(ARCH_DIR "x86")
+  endif()
+
+  find_library(
+        KINECT_LIBRARY_RELEASE Kinect20 
+        PATH_SUFFIXES "Lib/${ARCH_DIR}" "lib" 
+        HINTS ${KINECT_SEARCH_DIRS}
+        PATH $ENV{KINECT_ROOT_DIR})
+
+  set(KINECT_LIBRARIES ${KINECT_LIBRARY})
+
+  # DLL not needed yet??
+  #find_path(KINECT_DLL_PATH Kinect20.Face.dll PATH_SUFFIXES "bin" HINTS ${KINECT_SEARCH_DIRS})
+
+
+endif ()
+
+include(SelectLibraryConfigurations)
+select_library_configurations(KINECT)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(KINECT DEFAULT_MSG KINECT_INCLUDE_DIRS KINECT_LIBRARY)
+
+# DLLs not needed yet
+#if (WIN32)
+#  add_paths_to_fixup_libs(${KINECT_DLL_PATH})
+#endif ()
+
+mark_as_advanced(KINECT_INCLUDE_DIRS KINECT_LIBRARIES KINECT_SEARCH_DIRS)

interface/resources/controllers/kinect.json

@@ -0,0 +1,7 @@
+{
+    "name": "Kinect to Standard",
+    "channels": [
+        { "from": "Kinect.LeftHand", "to": "Standard.LeftHand" },
+        { "from": "Kinect.RightHand", "to": "Standard.RightHand" }
+    ]
+}

interface/resources/qml/hifi/dialogs/GeneralPreferencesDialog.qml

@@ -17,7 +17,7 @@ PreferencesDialog {
     id: root
     objectName: "GeneralPreferencesDialog"
     title: "General Settings"
-    showCategories: ["UI", "Snapshots", "Scripts", "Privacy", "Octree", "HMD", "Sixense Controllers", "Perception Neuron"]
+    showCategories: ["UI", "Snapshots", "Scripts", "Privacy", "Octree", "HMD", "Sixense Controllers", "Perception Neuron", "Kinect"]
     property var settings: Settings {
         category: root.objectName
         property alias x: root.x

libraries/controllers/src/controllers/impl/endpoints/StandardEndpoint.h

@@ -44,7 +44,7 @@ public:
     }
 
     virtual void apply(const Pose& value, const Pointer& source) override {
-        if (value != Pose()) {
+        if (value != Pose() && value.isValid()) {
             _written = true;
         }
         VirtualEndpoint::apply(value, source);

plugins/CMakeLists.txt

@@ -26,6 +26,9 @@ if (NOT SERVER_ONLY AND NOT ANDROID)
   add_subdirectory(${DIR})
   set(DIR "hifiNeuron")
   add_subdirectory(${DIR})
+
+  set(DIR "hifiKinect")
+  add_subdirectory(${DIR})
 endif()
 
 # server-side plugins

plugins/hifiKinect/CMakeLists.txt

@@ -0,0 +1,19 @@
+#
+#  Created by Brad Hefta-Gaub on 2016/12/7
+#  Copyright 2016 High Fidelity, Inc.
+#
+#  Distributed under the Apache License, Version 2.0.
+#  See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
+#
+
+if (WIN32)
+    find_package(KINECT)
+    if (KINECT_FOUND) 
+        set(TARGET_NAME hifiKinect)
+        setup_hifi_plugin(Script Qml Widgets)
+        link_hifi_libraries(shared controllers ui plugins input-plugins)
+    
+        # need to setup appropriate externals...
+        target_kinect()
+    endif()
+endif()

plugins/hifiKinect/src/KinectPlugin.cpp

@@ -0,0 +1,532 @@
+//
+//  KinectPlugin.cpp
+//
+//  Created by Brad Hefta-Gaub on 2016/12/7
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+
+#include "KinectPlugin.h"
+
+#include <controllers/UserInputMapper.h>
+#include <QLoggingCategory>
+#include <PathUtils.h>
+#include <DebugDraw.h>
+#include <cassert>
+#include <NumericalConstants.h>
+#include <StreamUtils.h>
+#include <Preferences.h>
+#include <SettingHandle.h>
+
+Q_DECLARE_LOGGING_CATEGORY(inputplugins)
+Q_LOGGING_CATEGORY(inputplugins, "hifi.inputplugins")
+
+
+const char* KinectPlugin::NAME = "Kinect";
+const char* KinectPlugin::KINECT_ID_STRING = "Kinect";
+
+QStringList kinectJointNames = {
+    "SpineBase",
+    "SpineMid",
+    "Neck",
+    "Head",
+    "ShoulderLeft",
+    "ElbowLeft",
+    "WristLeft",
+    "HandLeft",
+    "ShoulderRight",
+    "ElbowRight",
+    "WristRight",
+    "HandRight",
+    "HipLeft",
+    "KneeLeft",
+    "AnkleLeft",
+    "FootLeft",
+    "HipRight",
+    "KneeRight",
+    "AnkleRight",
+    "FootRight",
+    "SpineShoulder",
+    "HandTipLeft",
+    "ThumbLeft",
+    "HandTipRight",
+    "ThumbRight"
+};
+
+const bool DEFAULT_ENABLED = false;
+
+enum KinectJointIndex {
+    SpineBase = 0,
+    SpineMid,
+    Neck,
+    Head,
+
+    ShoulderLeft,
+    ElbowLeft,
+    WristLeft,
+    HandLeft,
+
+    ShoulderRight,
+    ElbowRight,
+    WristRight,
+    HandRight,
+
+    HipLeft,
+    KneeLeft,
+    AnkleLeft,
+    FootLeft,
+
+    HipRight,
+    KneeRight,
+    AnkleRight,
+    FootRight,
+
+    SpineShoulder,
+
+    HandTipLeft,
+    ThumbLeft,
+
+    HandTipRight,
+    ThumbRight,
+
+    Size
+};
+
+#define UNKNOWN_JOINT (controller::StandardPoseChannel)0 
+
+static controller::StandardPoseChannel KinectJointIndexToPoseIndexMap[KinectJointIndex::Size] = {
+    controller::HIPS,
+    controller::SPINE,
+    controller::NECK,
+    controller::HEAD,
+
+    controller::LEFT_SHOULDER,
+    controller::LEFT_ARM,
+    controller::LEFT_FORE_ARM,
+    controller::LEFT_HAND,
+
+    controller::RIGHT_SHOULDER,
+    controller::RIGHT_ARM,
+    controller::RIGHT_FORE_ARM,
+    controller::RIGHT_HAND,
+
+    controller::RIGHT_UP_LEG,   // hip socket
+    controller::RIGHT_LEG,      // knee?
+    controller::RIGHT_FOOT,     // ankle?
+    UNKNOWN_JOINT,              // ????
+
+    controller::LEFT_UP_LEG,   // hip socket
+    controller::LEFT_LEG,      // knee?
+    controller::LEFT_FOOT,     // ankle?
+    UNKNOWN_JOINT,              // ????
+
+    UNKNOWN_JOINT, /* SpineShoulder */
+
+    controller::LEFT_HAND_INDEX4,
+    controller::LEFT_HAND_THUMB4,
+
+    controller::RIGHT_HAND_INDEX4,
+    controller::RIGHT_HAND_THUMB4,
+
+};
+
+// in rig frame
+static glm::vec3 rightHandThumb1DefaultAbsTranslation(-2.155500650405884, -0.7610001564025879, 2.685631036758423);
+static glm::vec3 leftHandThumb1DefaultAbsTranslation(2.1555817127227783, -0.7603635787963867, 2.6856393814086914);
+
+static controller::StandardPoseChannel KinectJointIndexToPoseIndex(KinectJointIndex i) {
+    assert(i >= 0 && i < KinectJointIndex::Size);
+    if (i >= 0 && i < KinectJointIndex::Size) {
+        return KinectJointIndexToPoseIndexMap[i];
+    } else {
+        return UNKNOWN_JOINT; // not sure what to do here, but don't crash!
+    }
+}
+
+QStringList controllerJointNames = {
+    "Hips",
+    "RightUpLeg",
+    "RightLeg",
+    "RightFoot",
+    "LeftUpLeg",
+    "LeftLeg",
+    "LeftFoot",
+    "Spine",
+    "Spine1",
+    "Spine2",
+    "Spine3",
+    "Neck",
+    "Head",
+    "RightShoulder",
+    "RightArm",
+    "RightForeArm",
+    "RightHand",
+    "RightHandThumb1",
+    "RightHandThumb2",
+    "RightHandThumb3",
+    "RightHandThumb4",
+    "RightHandIndex1",
+    "RightHandIndex2",
+    "RightHandIndex3",
+    "RightHandIndex4",
+    "RightHandMiddle1",
+    "RightHandMiddle2",
+    "RightHandMiddle3",
+    "RightHandMiddle4",
+    "RightHandRing1",
+    "RightHandRing2",
+    "RightHandRing3",
+    "RightHandRing4",
+    "RightHandPinky1",
+    "RightHandPinky2",
+    "RightHandPinky3",
+    "RightHandPinky4",
+    "LeftShoulder",
+    "LeftArm",
+    "LeftForeArm",
+    "LeftHand",
+    "LeftHandThumb1",
+    "LeftHandThumb2",
+    "LeftHandThumb3",
+    "LeftHandThumb4",
+    "LeftHandIndex1",
+    "LeftHandIndex2",
+    "LeftHandIndex3",
+    "LeftHandIndex4",
+    "LeftHandMiddle1",
+    "LeftHandMiddle2",
+    "LeftHandMiddle3",
+    "LeftHandMiddle4",
+    "LeftHandRing1",
+    "LeftHandRing2",
+    "LeftHandRing3",
+    "LeftHandRing4",
+    "LeftHandPinky1",
+    "LeftHandPinky2",
+    "LeftHandPinky3",
+    "LeftHandPinky4"
+};
+
+static const char* getControllerJointName(controller::StandardPoseChannel i) {
+    if (i >= 0 && i < controller::NUM_STANDARD_POSES) {
+        return qPrintable(controllerJointNames[i]);
+    }
+    return "unknown";
+}
+
+//
+// KinectPlugin
+//
+void KinectPlugin::init() {
+    loadSettings();
+
+    auto preferences = DependencyManager::get<Preferences>();
+    static const QString KINECT_PLUGIN { "Kinect" };
+    {
+        auto getter = [this]()->bool { return _enabled; };
+        auto setter = [this](bool value) { _enabled = value; saveSettings(); };
+        auto preference = new CheckPreference(KINECT_PLUGIN, "Enabled", getter, setter);
+        preferences->addPreference(preference);
+    }
+}
+
+bool KinectPlugin::isSupported() const {
+    // FIXME -- check to see if there's a camera or not...
+    return true;
+}
+
+bool KinectPlugin::activate() {
+    InputPlugin::activate();
+
+    loadSettings();
+
+    if (_enabled) {
+
+        // register with userInputMapper
+        auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
+        userInputMapper->registerDevice(_inputDevice);
+
+        return initializeDefaultSensor();
+    } else {
+        return false;
+    }
+}
+
+bool KinectPlugin::initializeDefaultSensor() {
+#ifdef HAVE_KINECT
+    HRESULT hr;
+
+    hr = GetDefaultKinectSensor(&_kinectSensor);
+    if (FAILED(hr)) {
+        return false;
+    }
+
+    if (_kinectSensor) {
+        // Initialize the Kinect and get coordinate mapper and the body reader
+        IBodyFrameSource* bodyFrameSource = NULL;
+
+        hr = _kinectSensor->Open();
+
+        if (SUCCEEDED(hr)) {
+            hr = _kinectSensor->get_CoordinateMapper(&_coordinateMapper);
+        }
+
+        if (SUCCEEDED(hr)) {
+            hr = _kinectSensor->get_BodyFrameSource(&bodyFrameSource);
+        }
+
+        if (SUCCEEDED(hr)) {
+            hr = bodyFrameSource->OpenReader(&_bodyFrameReader);
+        }
+
+        SafeRelease(bodyFrameSource);
+    }
+
+    if (!_kinectSensor || FAILED(hr)) {
+        qDebug() << "No ready Kinect found!";
+        return false;
+    }
+
+    qDebug() << "Kinect found WOOT!";
+
+    return true;
+#else
+    return false;
+#endif
+}
+
+void KinectPlugin::updateBody() {
+#ifndef HAVE_KINECT
+    return;
+#else
+    if (!_bodyFrameReader) {
+        return;
+    }
+
+    IBodyFrame* pBodyFrame = NULL;
+
+    HRESULT hr = _bodyFrameReader->AcquireLatestFrame(&pBodyFrame);
+
+    if (SUCCEEDED(hr)) {
+        INT64 nTime = 0;
+        hr = pBodyFrame->get_RelativeTime(&nTime);
+        IBody* bodies[BODY_COUNT] = {0};
+        if (SUCCEEDED(hr)) {
+            hr = pBodyFrame->GetAndRefreshBodyData(_countof(bodies), bodies);
+        }
+
+        if (SUCCEEDED(hr)) {
+            ProcessBody(nTime, BODY_COUNT, bodies);
+        }
+
+        for (int i = 0; i < _countof(bodies); ++i) {
+            SafeRelease(bodies[i]);
+        }
+    }
+
+    SafeRelease(pBodyFrame);
+#endif
+}
+
+#ifdef HAVE_KINECT
+void KinectPlugin::ProcessBody(INT64 time, int bodyCount, IBody** bodies) {
+    bool foundOneBody = false;
+    if (_coordinateMapper) {
+        for (int i = 0; i < bodyCount; ++i) {
+            if (foundOneBody) {
+                break;
+            }
+            IBody* body = bodies[i];
+            if (body) {
+                BOOLEAN tracked = false;
+                HRESULT hr = body->get_IsTracked(&tracked);
+
+                if (SUCCEEDED(hr) && tracked) {
+                    foundOneBody = true;
+
+                    if (_joints.size() != JointType_Count) {
+                        _joints.resize(JointType_Count, { { 0.0f, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f, 0.0f } });
+                    }
+
+                    Joint joints[JointType_Count];
+                    JointOrientation jointOrientations[JointType_Count];
+                    HandState leftHandState = HandState_Unknown;
+                    HandState rightHandState = HandState_Unknown;
+
+                    body->get_HandLeftState(&leftHandState);
+                    body->get_HandRightState(&rightHandState);
+
+                    hr = body->GetJoints(_countof(joints), joints);
+                    hr = body->GetJointOrientations(_countof(jointOrientations), jointOrientations);
+
+                    if (SUCCEEDED(hr)) {
+                        auto jointCount = _countof(joints);
+                        //qDebug() << __FUNCTION__ << "nBodyCount:" << nBodyCount << "body:" << i << "jointCount:" << jointCount;
+                        for (int j = 0; j < jointCount; ++j) {
+                            //QString jointName = kinectJointNames[joints[j].JointType];
+
+                            glm::vec3 jointPosition { joints[j].Position.X,
+                                                      joints[j].Position.Y,
+                                                      joints[j].Position.Z };
+
+                            // Kinect Documentation is unclear on what these orientations are, are they absolute? 
+                            // or are the relative to the parent bones. It appears as if it has changed between the
+                            // older 1.x SDK and the 2.0 sdk
+                            //
+                            // https://social.msdn.microsoft.com/Forums/en-US/31c9aff6-7dab-433d-9af9-59942dfd3d69/kinect-v20-preview-sdk-jointorientation-vs-boneorientation?forum=kinectv2sdk
+                            // seems to suggest these are absolute...
+                            //    "These quaternions are absolute, so you can take a mesh in local space, transform it by the quaternion, 
+                            //    and it will match the exact orientation of the bone.  If you want relative orientation quaternion, you 
+                            //    can multiply the absolute quaternion by the inverse of the parent joint's quaternion."
+                            //
+                            //  - Bone direction(Y green) - always matches the skeleton.
+                            //  - Normal(Z blue) - joint roll, perpendicular to the bone
+                            //  - Binormal(X orange) - perpendicular to the bone and normal
+
+                            glm::quat jointOrientation { jointOrientations[j].Orientation.x,
+                                                         jointOrientations[j].Orientation.y,
+                                                         jointOrientations[j].Orientation.z,
+                                                         jointOrientations[j].Orientation.w };
+
+                            // filling in the _joints data...
+                            if (joints[j].TrackingState != TrackingState_NotTracked) {
+                                _joints[j].position = jointPosition;
+                                //_joints[j].orientation = jointOrientation;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+#endif
+
+
+void KinectPlugin::deactivate() {
+    // unregister from userInputMapper
+    if (_inputDevice->_deviceID != controller::Input::INVALID_DEVICE) {
+        auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
+        userInputMapper->removeDevice(_inputDevice->_deviceID);
+    }
+
+    InputPlugin::deactivate();
+    saveSettings();
+
+#ifdef HAVE_KINECT
+    // done with body frame reader
+    SafeRelease(_bodyFrameReader);
+
+    // done with coordinate mapper
+    SafeRelease(_coordinateMapper);
+
+    // close the Kinect Sensor
+    if (_kinectSensor) {
+        _kinectSensor->Close();
+    }
+
+    SafeRelease(_kinectSensor);
+#endif // HAVE_KINECT
+
+}
+
+void KinectPlugin::pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
+
+    if (!_enabled) {
+        return;
+    }
+
+    updateBody(); // updates _joints
+
+    std::vector<KinectJoint> joints = _joints;
+
+    auto userInputMapper = DependencyManager::get<controller::UserInputMapper>();
+    userInputMapper->withLock([&, this]() {
+        _inputDevice->update(deltaTime, inputCalibrationData, joints, _prevJoints);
+    });
+
+    _prevJoints = joints;
+}
+
+void KinectPlugin::saveSettings() const {
+    Settings settings;
+    QString idString = getID();
+    settings.beginGroup(idString);
+    {
+        settings.setValue(QString("enabled"), _enabled);
+    }
+    settings.endGroup();
+}
+
+void KinectPlugin::loadSettings() {
+    Settings settings;
+    QString idString = getID();
+    settings.beginGroup(idString);
+    {
+        // enabled
+        _enabled = settings.value("enabled", QVariant(DEFAULT_ENABLED)).toBool();
+    }
+    settings.endGroup();
+}
+
+//
+// InputDevice
+//
+
+// FIXME - we probably should only return the inputs we ACTUALLY have
+controller::Input::NamedVector KinectPlugin::InputDevice::getAvailableInputs() const {
+    static controller::Input::NamedVector availableInputs;
+    if (availableInputs.size() == 0) {
+        for (int i = 0; i < KinectJointIndex::Size; i++) {
+            auto channel = KinectJointIndexToPoseIndex(static_cast<KinectJointIndex>(i));
+            availableInputs.push_back(makePair(channel, getControllerJointName(channel)));
+        }
+    };
+    return availableInputs;
+}
+
+QString KinectPlugin::InputDevice::getDefaultMappingConfig() const {
+    static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/kinect.json";
+    return MAPPING_JSON;
+}
+
+void KinectPlugin::InputDevice::update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData, 
+                                        const std::vector<KinectPlugin::KinectJoint>& joints, const std::vector<KinectPlugin::KinectJoint>& prevJoints) {
+
+    glm::mat4 controllerToAvatar = glm::inverse(inputCalibrationData.avatarMat) * inputCalibrationData.sensorToWorldMat;
+    glm::quat controllerToAvatarRotation = glmExtractRotation(controllerToAvatar);
+
+    vec3 kinectHipPos;
+    if (joints.size() > JointType_SpineBase) {
+        kinectHipPos = joints[JointType_SpineBase].position;
+    }
+
+    for (size_t i = 0; i < joints.size(); i++) {
+        int poseIndex = KinectJointIndexToPoseIndex((KinectJointIndex)i);
+        glm::vec3 linearVel, angularVel;
+
+        // Adjust the position to be hip (avatar) relative, and rotated to match the avatar rotation
+        const glm::vec3& pos = controllerToAvatarRotation * (joints[i].position - kinectHipPos);
+
+        if (Vectors::ZERO == pos) {
+            _poseStateMap[poseIndex] = controller::Pose();
+            continue;
+        }
+
+        // FIXME - determine the correct orientation transform
+        glm::quat rot = joints[i].orientation;
+
+        if (i < prevJoints.size()) {
+            linearVel = (pos - (prevJoints[i].position * METERS_PER_CENTIMETER)) / deltaTime;  // m/s
+            // quat log imaginary part points along the axis of rotation, with length of one half the angle of rotation.
+            glm::quat d = glm::log(rot * glm::inverse(prevJoints[i].orientation));
+            angularVel = glm::vec3(d.x, d.y, d.z) / (0.5f * deltaTime); // radians/s
+        }
+
+        _poseStateMap[poseIndex] = controller::Pose(pos, rot, linearVel, angularVel);
+    }
+}
+

plugins/hifiKinect/src/KinectPlugin.h

@@ -0,0 +1,117 @@
+//
+//  KinectPlugin.h
+//
+//  Created by Brad Hefta-Gaub on 2016/12/7
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_KinectPlugin_h
+#define hifi_KinectPlugin_h
+
+#ifdef HAVE_KINECT
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers
+#endif
+
+// Windows Header Files
+#include <windows.h>
+
+#include <Shlobj.h>
+
+// Kinect Header files
+#include <Kinect.h>
+
+// Safe release for interfaces
+template<class Interface> inline void SafeRelease(Interface *& pInterfaceToRelease) {
+    if (pInterfaceToRelease != NULL) {
+        pInterfaceToRelease->Release();
+        pInterfaceToRelease = NULL;
+    }
+}
+#endif
+
+#include <controllers/InputDevice.h>
+#include <controllers/StandardControls.h>
+#include <plugins/InputPlugin.h>
+
+// Handles interaction with the Kinect SDK
+class KinectPlugin : public InputPlugin {
+    Q_OBJECT
+public:
+    bool isHandController() const override { return true; }
+
+    // Plugin functions
+    virtual void init() override;
+    virtual bool isSupported() const override;
+    virtual const QString getName() const override { return NAME; }
+    const QString getID() const override { return KINECT_ID_STRING; }
+
+    virtual bool activate() override;
+    virtual void deactivate() override;
+
+    virtual void pluginFocusOutEvent() override { _inputDevice->focusOutEvent(); }
+    virtual void pluginUpdate(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override;
+
+    virtual void saveSettings() const override;
+    virtual void loadSettings() override;
+
+protected:
+
+    struct KinectJoint {
+        glm::vec3 position;
+        glm::quat orientation;
+    };
+
+    class InputDevice : public controller::InputDevice {
+    public:
+        friend class KinectPlugin;
+
+        InputDevice() : controller::InputDevice("Kinect") {}
+
+        // Device functions
+        virtual controller::Input::NamedVector getAvailableInputs() const override;
+        virtual QString getDefaultMappingConfig() const override;
+        virtual void update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) override {};
+        virtual void focusOutEvent() override {};
+
+        void update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData, 
+                const std::vector<KinectPlugin::KinectJoint>& joints, const std::vector<KinectPlugin::KinectJoint>& prevJoints);
+    };
+
+    std::shared_ptr<InputDevice> _inputDevice { std::make_shared<InputDevice>() };
+
+    static const char* NAME;
+    static const char* KINECT_ID_STRING;
+
+    bool _enabled;
+
+    // copy of data directly from the KinectDataReader SDK
+    std::vector<KinectJoint> _joints;
+
+    // one frame old copy of _joints, used to caluclate angular and linear velocity.
+    std::vector<KinectJoint> _prevJoints;
+
+
+    // Kinect SDK related items...
+
+    bool KinectPlugin::initializeDefaultSensor();
+    void updateBody();
+
+#ifdef HAVE_KINECT
+    void ProcessBody(INT64 time, int bodyCount, IBody** bodies);
+
+    // Current Kinect
+    IKinectSensor*          _kinectSensor { nullptr };
+    ICoordinateMapper*      _coordinateMapper { nullptr };
+
+    // Body reader
+    IBodyFrameReader*       _bodyFrameReader { nullptr };
+#endif
+
+};
+
+#endif // hifi_KinectPlugin_h
+

plugins/hifiKinect/src/KinectProvider.cpp

@@ -0,0 +1,49 @@
+//
+//  Created by Brad Hefta-Gaub on 2016/12/7
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <mutex>
+
+#include <QtCore/QObject>
+#include <QtCore/QtPlugin>
+#include <QtCore/QStringList>
+
+#include <plugins/RuntimePlugin.h>
+#include <plugins/InputPlugin.h>
+
+#include "KinectPlugin.h"
+
+class KinectProvider : public QObject, public InputProvider
+{
+    Q_OBJECT
+    Q_PLUGIN_METADATA(IID InputProvider_iid FILE "plugin.json")
+    Q_INTERFACES(InputProvider)
+
+public:
+    KinectProvider(QObject* parent = nullptr) : QObject(parent) {}
+    virtual ~KinectProvider() {}
+
+    virtual InputPluginList getInputPlugins() override {
+        static std::once_flag once;
+        std::call_once(once, [&] {
+            InputPluginPointer plugin(new KinectPlugin());
+            if (plugin->isSupported()) {
+                _inputPlugins.push_back(plugin);
+            }
+        });
+        return _inputPlugins;
+    }
+
+    virtual void destroyInputPlugins() override {
+        _inputPlugins.clear();
+    }
+
+private:
+    InputPluginList _inputPlugins;
+};
+
+#include "KinectProvider.moc"

plugins/hifiKinect/src/plugin.json

@@ -0,0 +1 @@
+{"name":"Kinect"}