830 lines
32 KiB
JavaScript
830 lines
32 KiB
JavaScript
"use strict";
|
|
/*jslint vars:true, plusplus:true, forin:true*/
|
|
/*global Tablet, Script, */
|
|
/* eslint indent: ["error", 4, { "outerIIFEBody": 0 }] */
|
|
//
|
|
// gestures.js
|
|
//
|
|
// Created by Zach Fox on 2018-07-24
|
|
// Copyright 2018 High Fidelity, Inc
|
|
//
|
|
// Distributed under the Apache License, Version 2.0
|
|
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
|
//
|
|
|
|
(function () { // BEGIN LOCAL_SCOPE
|
|
var AppUi = Script.require('appUi');
|
|
|
|
/********************************
|
|
// START Debug Functions
|
|
********************************/
|
|
var DEBUG_UNIMPORTANT = 0;
|
|
var DEBUG_IMPORTANT = 1;
|
|
var DEBUG_URGENT = 2;
|
|
|
|
var DEBUG_ONLY_PRINT_URGENT = 0;
|
|
var DEBUG_PRINT_URGENT_AND_IMPORTANT = 1;
|
|
var DEBUG_PRINT_EVERYTHING = 2;
|
|
var debugLevel = DEBUG_PRINT_URGENT_AND_IMPORTANT;
|
|
|
|
function maybePrint(string, importance) {
|
|
if (importance >= (DEBUG_URGENT - debugLevel)) {
|
|
console.log(string);
|
|
}
|
|
}
|
|
/********************************
|
|
// END Debug Functions
|
|
********************************/
|
|
|
|
/********************************
|
|
// START GestureRecorder
|
|
********************************/
|
|
var LEFT_HAND_JOINT_NAME = "leftHand";
|
|
var RIGHT_HAND_JOINT_NAME = "rightHand";
|
|
var HEAD_JOINT_NAME = "head";
|
|
function GestureRecorder(jointName) {
|
|
this.jointName = jointName;
|
|
this.currentPoseFrameData = [];
|
|
this.previouslyRecordedFrameData = [];
|
|
this.recordingStartTimeMS;
|
|
this.recordingLengthSEC;
|
|
this.isRecordingGesture = false;
|
|
}
|
|
|
|
GestureRecorder.prototype.initializeDataForRecording = function () {
|
|
this.currentPoseFrameData = [];
|
|
this.recordingStartTimeMS = Date.now();
|
|
maybePrint("ZRF: Starting gesture recording for joint '" + this.jointName + "'.", DEBUG_IMPORTANT);
|
|
}
|
|
|
|
var Y_AXIS = { x: 0, y: 1, z: 0 };
|
|
GestureRecorder.prototype.sampleFrame = function (timeSinceStartSEC, previousSample) {
|
|
var pose;
|
|
if (this.jointName === LEFT_HAND_JOINT_NAME) {
|
|
pose = Controller.getPoseValue(Controller.Standard.LeftHand);
|
|
} else if (this.jointName === RIGHT_HAND_JOINT_NAME) {
|
|
pose = Controller.getPoseValue(Controller.Standard.RightHand);
|
|
} else if (this.jointName === HEAD_JOINT_NAME) {
|
|
pose = Controller.getPoseValue(Controller.Standard.Head);
|
|
}
|
|
|
|
var frameData = {
|
|
timeSinceStartSEC: timeSinceStartSEC,
|
|
x: pose.translation.x,
|
|
y: pose.translation.y,
|
|
z: pose.translation.z,
|
|
rotation: pose.rotation
|
|
};
|
|
|
|
return frameData;
|
|
}
|
|
|
|
var MS_PER_SEC = 1000;
|
|
GestureRecorder.prototype.captureDataNow = function () {
|
|
var now = Date.now();
|
|
var timeSinceStartSEC = (now - this.recordingStartTimeMS) / MS_PER_SEC;
|
|
this.currentPoseFrameData.push(this.sampleFrame(timeSinceStartSEC));
|
|
}
|
|
|
|
GestureRecorder.prototype.stopRecording = function () {
|
|
this.recordingLengthSEC = (Date.now() - this.recordingStartTimeMS) / MS_PER_SEC;
|
|
calculateDerivatives(this.currentPoseFrameData);
|
|
maybePrint("ZRF: Finished gesture recording for joint '" + this.jointName + "'.", DEBUG_IMPORTANT);
|
|
JSON.stringify(this.currentPoseFrameData, null, 4).split("\n").forEach(function (str) {
|
|
maybePrint(str, DEBUG_UNIMPORTANT);
|
|
});
|
|
}
|
|
|
|
GestureRecorder.prototype.gestureDetectCheck = function () {
|
|
maybePrint("gestureDetect() currentPoseFrameData.length = " + this.currentPoseFrameData.length +
|
|
", previouslyRecordedFrameData.data.length = " + this.previouslyRecordedFrameData[gestureToDetect_index].data.length +
|
|
", recordingLength = " + this.recordingLengthSEC + " (sec)", DEBUG_UNIMPORTANT);
|
|
|
|
// not enough frames to test
|
|
if (this.currentPoseFrameData.length < this.previouslyRecordedFrameData[gestureToDetect_index].data.length / 2) {
|
|
return false;
|
|
}
|
|
|
|
var i = 0, j = 0;
|
|
var framesTested = 0;
|
|
var framesPassed = 0;
|
|
while (i < this.previouslyRecordedFrameData[gestureToDetect_index].data.length && j < this.currentPoseFrameData.length) {
|
|
var it = this.previouslyRecordedFrameData[gestureToDetect_index].data[i].timeSinceStartSEC;
|
|
var jt = this.currentPoseFrameData[j].timeSinceStartSEC - this.currentPoseFrameData[0].timeSinceStartSEC;
|
|
if (jt < it) {
|
|
var iPrev = i > 0 ? (i - 1) : 0;
|
|
var a = this.previouslyRecordedFrameData[gestureToDetect_index].data[iPrev];
|
|
var b = this.previouslyRecordedFrameData[gestureToDetect_index].data[i];
|
|
var alpha = i > 0 ? (jt - a.timeSinceStartSEC) / (b.timeSinceStartSEC - a.timeSinceStartSEC) : 0;
|
|
var frame = lerpFrame(a, b, alpha);
|
|
framesTested++;
|
|
if (testFrames(this.currentPoseFrameData[j], frame)) {
|
|
framesPassed++;
|
|
}
|
|
j++;
|
|
} else {
|
|
i++;
|
|
}
|
|
}
|
|
|
|
if (framesPassed / framesTested > 0.4) {
|
|
maybePrint(~~((framesPassed / framesTested) * 100) + "% match", DEBUG_UNIMPORTANT);
|
|
}
|
|
|
|
return framesPassed / framesTested > 0.75;
|
|
}
|
|
|
|
GestureRecorder.prototype.gestureDetect = function () {
|
|
var t = Date.now() / MS_PER_SEC;
|
|
this.currentPoseFrameData.push(this.sampleFrame(t));
|
|
calculateDerivatives(this.currentPoseFrameData);
|
|
|
|
var i;
|
|
for (i = 0; i < this.currentPoseFrameData.length; i++) {
|
|
if (i > 0 && this.currentPoseFrameData[i].timeSinceStartSEC > t - this.recordingLengthSEC) {
|
|
this.currentPoseFrameData = this.currentPoseFrameData.slice(i - 1);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (this.currentPoseFrameData.length > 0 && this.gestureDetectCheck()) {
|
|
maybePrint("ZRF: Gesture with index " + gestureToDetect_index + " detected on joint " + this.jointName + "!", DEBUG_UNIMPORTANT);
|
|
return true;
|
|
}
|
|
return false;
|
|
};
|
|
/********************************
|
|
// END GestureRecorder
|
|
********************************/
|
|
|
|
/********************************
|
|
// START Shared Math Utility Functions
|
|
********************************/
|
|
// Function Name: inFrontOf()
|
|
//
|
|
// Description:
|
|
// -Returns the position in front of the given "position" argument, where the forward vector is based off
|
|
// the "orientation" argument and the amount in front is based off the "distance" argument.
|
|
function inFrontOf(distance, position, orientation) {
|
|
return Vec3.sum(position || MyAvatar.position,
|
|
Vec3.multiply(distance, Quat.getForward(orientation || MyAvatar.orientation)));
|
|
}
|
|
|
|
function calculateDerivatives(frames) {
|
|
var i, length = frames.length;
|
|
var keys = ["x", "y", "z"];
|
|
var dKeys = ["dx", "dy", "dz"];
|
|
|
|
for (i = 0; i < length; i++) {
|
|
var prevIndex = (i === 0) ? 0 : i - 1;
|
|
var nextIndex = (i === length - 1) ? i : i + 1;
|
|
var j = 0, numKeys = keys.length;
|
|
for (j = 0; j < numKeys; j++) {
|
|
var d1 = frames[i][keys[j]] - frames[prevIndex][keys[j]];
|
|
var d2 = frames[nextIndex][keys[j]] - frames[i][keys[j]];
|
|
frames[i][dKeys[j]] = (d1 + d2) / 2;
|
|
}
|
|
}
|
|
}
|
|
|
|
function lerp(a, b, alpha) {
|
|
return a * (1 - alpha) + b * alpha;
|
|
}
|
|
|
|
function lerpFrame(a, b, alpha) {
|
|
var keys = Object.keys(a);
|
|
var result = {};
|
|
keys.forEach(function (key) {
|
|
result[key] = lerp(a[key], b[key], alpha);
|
|
});
|
|
return result;
|
|
}
|
|
|
|
var RAD_TO_DEG = 180 / Math.PI;
|
|
var DEG_TO_RAD = Math.PI / 180;
|
|
var ROTATION_THRESHOLD = 3.0 * DEG_TO_RAD; // radians
|
|
var X_THRESHOLD = 0.2; // meters
|
|
var Y_THRESHOLD = 0.2; // meters
|
|
var Z_THRESHOLD = 0.2; // meters
|
|
var DX_THRESHOLD = 0.02; // meters / sec
|
|
var DY_THRESHOLD = 0.02; // meters / sec
|
|
var DZ_THRESHOLD = 0.02; // meters / sec
|
|
function testFrames(a, b) {
|
|
if (Math.abs(Quat.dot(a.rotation, b.rotation)) > Math.cos(ROTATION_THRESHOLD * gestureDetectionSensitivityMultiplier / 2.0)) {
|
|
return false;
|
|
}
|
|
if (Math.abs(a.x - b.x) > X_THRESHOLD * gestureDetectionSensitivityMultiplier) {
|
|
return false;
|
|
}
|
|
if (Math.abs(a.y - b.y) > Y_THRESHOLD * gestureDetectionSensitivityMultiplier) {
|
|
return false;
|
|
}
|
|
if (Math.abs(a.z - b.z) > Z_THRESHOLD * gestureDetectionSensitivityMultiplier) {
|
|
return false;
|
|
}
|
|
if (Math.abs(a.dx - b.dx) > DX_THRESHOLD * gestureDetectionSensitivityMultiplier) {
|
|
return false;
|
|
}
|
|
if (Math.abs(a.dy - b.dy) > DY_THRESHOLD * gestureDetectionSensitivityMultiplier) {
|
|
return false;
|
|
}
|
|
if (Math.abs(a.dz - b.dz) > DZ_THRESHOLD * gestureDetectionSensitivityMultiplier) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
/********************************
|
|
// END Shared Math Utility Functions
|
|
********************************/
|
|
|
|
/********************************
|
|
// START Global Detection Start/Stop/Update functions
|
|
********************************/
|
|
var isDetectingGesture = false;
|
|
var gestureToDetect_index = -1;
|
|
var gestureDetectionSensitivityMultiplier = Settings.getValue('gestures/sensitivity', 1.0);
|
|
function updateGestureDetectionSystem() {
|
|
if (gestureToDetect_index > -1 && !isRecordingSelectedGestures) {
|
|
if (isDetectingGesture) {
|
|
Script.update.disconnect(gestureDetectionUpdateLoop);
|
|
isDetectingGesture = false;
|
|
}
|
|
|
|
isDetectingGesture = true;
|
|
Script.update.connect(gestureDetectionUpdateLoop);
|
|
} else {
|
|
if (isDetectingGesture) {
|
|
Script.update.disconnect(gestureDetectionUpdateLoop);
|
|
isDetectingGesture = false;
|
|
|
|
leftHandRecorder.currentPoseFrameData = [];
|
|
rightHandRecorder.currentPoseFrameData = [];
|
|
headRecorder.currentPoseFrameData = [];
|
|
}
|
|
}
|
|
|
|
maybePrint("ZRF: The gesture to detect has index: " + gestureToDetect_index + ". Currently detecting gestures: " + isDetectingGesture, DEBUG_IMPORTANT);
|
|
}
|
|
|
|
var detectedEntity = false;
|
|
var deleteDetectedEntityTimeout = false;
|
|
function handleDetectedEntity() {
|
|
if (!detectedEntity) {
|
|
detectedEntity = Entities.addEntity({
|
|
"collidesWith": "",
|
|
"collisionMask": 0,
|
|
"collisionless": true,
|
|
"color": {
|
|
"blue": 20,
|
|
"green": 200,
|
|
"red": 20
|
|
},
|
|
"dimensions": {
|
|
"blue": 0.05000000074505806,
|
|
"green": 0.4000000059604645,
|
|
"red": 0.4000000059604645,
|
|
"x": 0.4000000059604645,
|
|
"y": 0.4000000059604645,
|
|
"z": 0.05000000074505806
|
|
},
|
|
"ignoreForCollisions": true,
|
|
"shape": "Cube",
|
|
"type": "Box",
|
|
"userData": "{\"grabbableKey\":{\"grabbable\":false}}",
|
|
"position": inFrontOf(0.8, Camera.position, Camera.orientation),
|
|
"rotation": Camera.orientation
|
|
}, true);
|
|
}
|
|
|
|
if (deleteDetectedEntityTimeout) {
|
|
Script.clearTimeout(deleteDetectedEntityTimeout);
|
|
}
|
|
|
|
deleteDetectedEntityTimeout = Script.setTimeout(function () {
|
|
if (detectedEntity) {
|
|
Entities.deleteEntity(detectedEntity);
|
|
detectedEntity = false;
|
|
}
|
|
deleteDetectedEntityTimeout = false;
|
|
}, 1000);
|
|
}
|
|
|
|
function handleEntityPickRay() {
|
|
var pickRay = {
|
|
origin: Camera.position,
|
|
direction: Quat.getFront(Camera.orientation),
|
|
length: 100
|
|
}
|
|
var entityIntersection = Entities.findRayIntersection(pickRay, true);
|
|
|
|
if (entityIntersection.intersects) {
|
|
var intersectEntityID = entityIntersection.entityID;
|
|
Entities.editEntity(intersectEntityID, {
|
|
color: {
|
|
red: Math.random() * 255,
|
|
green: Math.random() * 255,
|
|
blue: Math.random() * 255
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
function handleAvatarPickRay() {
|
|
var pickRay = {
|
|
origin: MyAvatar.position,
|
|
direction: Quat.getFront(Camera.orientation),
|
|
length: avatarIgnoreMaxDistance
|
|
}
|
|
var avatarIntersection = AvatarList.findRayIntersection(pickRay, [], [MyAvatar.sessionUUID]);
|
|
|
|
if (avatarIntersection.intersects) {
|
|
var avatarID = avatarIntersection.avatarID;
|
|
Users.ignore(avatarID);
|
|
} else {
|
|
pickRay.origin = Camera.position;
|
|
avatarIntersection = AvatarList.findRayIntersection(pickRay, [], [MyAvatar.sessionUUID]);
|
|
|
|
if (avatarIntersection.intersects) {
|
|
avatarID = avatarIntersection.avatarID;
|
|
Users.ignore(avatarID);
|
|
} else {
|
|
pickRay.origin.y += (Camera.position.y - MyAvatar.position.y);
|
|
avatarIntersection = AvatarList.findRayIntersection(pickRay, [], [MyAvatar.sessionUUID]);
|
|
|
|
if (avatarIntersection.intersects) {
|
|
avatarID = avatarIntersection.avatarID;
|
|
Users.ignore(avatarID);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function performGestureDetectedAction() {
|
|
maybePrint("ZRF: GESTURE WITH INDEX " + gestureToDetect_index + " DETECTED!", DEBUG_URGENT);
|
|
sendToQml({ method: 'gestureDetected' });
|
|
|
|
Audio.playSound(SOUND_GESTURE_DETECTED, {
|
|
position: MyAvatar.position,
|
|
localOnly: true,
|
|
volume: 0.3
|
|
});
|
|
|
|
//handleDetectedEntity();
|
|
|
|
handleEntityPickRay();
|
|
|
|
handleAvatarPickRay();
|
|
}
|
|
|
|
function gestureDetectionUpdateLoop() {
|
|
var detected = [false, false, false];
|
|
var mustBeDetected = [
|
|
leftHandRecorder.previouslyRecordedFrameData[gestureToDetect_index] !== "NODATA",
|
|
rightHandRecorder.previouslyRecordedFrameData[gestureToDetect_index] !== "NODATA",
|
|
headRecorder.previouslyRecordedFrameData[gestureToDetect_index] !== "NODATA"
|
|
];
|
|
|
|
if (mustBeDetected[0]) {
|
|
detected[0] = leftHandRecorder.gestureDetect();
|
|
}
|
|
if (mustBeDetected[1]) {
|
|
detected[1] = rightHandRecorder.gestureDetect();
|
|
}
|
|
if (mustBeDetected[2]) {
|
|
detected[2] = headRecorder.gestureDetect();
|
|
}
|
|
|
|
var allNecessaryGesturesDetected = false;
|
|
for (var i = 0; i < 3; i++) {
|
|
if (detected[i] !== mustBeDetected[i]) {
|
|
allNecessaryGesturesDetected = false;
|
|
break;
|
|
} else {
|
|
allNecessaryGesturesDetected = true;
|
|
}
|
|
}
|
|
|
|
if (allNecessaryGesturesDetected) {
|
|
performGestureDetectedAction();
|
|
|
|
leftHandRecorder.currentPoseFrameData = [];
|
|
rightHandRecorder.currentPoseFrameData = [];
|
|
headRecorder.currentPoseFrameData = [];
|
|
}
|
|
}
|
|
/********************************
|
|
// END Global Detection Start/Stop/Update functions
|
|
********************************/
|
|
|
|
/********************************
|
|
// START Global Capture Start/Stop/Update functions
|
|
********************************/
|
|
var dataCaptureStartTimeMS;
|
|
var DATA_CAPTURE_TIMEOUT_MS = 3000;
|
|
function dataCaptureUpdateLoop() {
|
|
if (Date.now() - dataCaptureStartTimeMS > DATA_CAPTURE_TIMEOUT_MS) {
|
|
stopRecordingSelectedGestures();
|
|
return;
|
|
}
|
|
if (jointDataToRecord[0]) {
|
|
leftHandRecorder.captureDataNow();
|
|
}
|
|
if (jointDataToRecord[1]) {
|
|
rightHandRecorder.captureDataNow();
|
|
}
|
|
if (jointDataToRecord[2]) {
|
|
headRecorder.captureDataNow();
|
|
}
|
|
}
|
|
|
|
// [0] is "left hand"
|
|
// [1] is "right hand"
|
|
// [2] is "head"
|
|
var jointDataToRecord = [
|
|
Settings.getValue('gestures/captureLeftHandData', false),
|
|
Settings.getValue('gestures/captureRightHandData', false),
|
|
Settings.getValue('gestures/captureHeadData', false)
|
|
];
|
|
var updateConnected = false;
|
|
var leftHandRecorder = new GestureRecorder(LEFT_HAND_JOINT_NAME);
|
|
var rightHandRecorder = new GestureRecorder(RIGHT_HAND_JOINT_NAME);
|
|
var headRecorder = new GestureRecorder(HEAD_JOINT_NAME);
|
|
function startRecordingSelectedGestures() {
|
|
if (!(jointDataToRecord[0] || jointDataToRecord[1] || jointDataToRecord[2])) {
|
|
return;
|
|
}
|
|
|
|
maybePrint("ZRF startRecordingSelectedGestures()", DEBUG_UNIMPORTANT);
|
|
isRecordingSelectedGestures = true;
|
|
sendToQml({ method: 'updateIsRecordingSelectedGestures', isRecordingSelectedGestures: isRecordingSelectedGestures });
|
|
Audio.playSound(SOUND_GESTURE_RECORDING_START, {
|
|
position: MyAvatar.position,
|
|
localOnly: true,
|
|
volume: 0.8
|
|
});
|
|
|
|
if (updateConnected) {
|
|
Script.update.disconnect(dataCaptureUpdateLoop);
|
|
updateConnected = false;
|
|
}
|
|
|
|
if (jointDataToRecord[0]) {
|
|
leftHandRecorder.initializeDataForRecording();
|
|
}
|
|
if (jointDataToRecord[1]) {
|
|
rightHandRecorder.initializeDataForRecording();
|
|
}
|
|
if (jointDataToRecord[2]) {
|
|
headRecorder.initializeDataForRecording();
|
|
}
|
|
|
|
dataCaptureStartTimeMS = Date.now();
|
|
Script.update.connect(dataCaptureUpdateLoop);
|
|
updateConnected = true;
|
|
|
|
updateGestureDetectionSystem();
|
|
}
|
|
|
|
function stopRecordingSelectedGestures() {
|
|
maybePrint("ZRF stopRecordingSelectedGestures()", DEBUG_UNIMPORTANT);
|
|
isRecordingSelectedGestures = false;
|
|
if (updateConnected) {
|
|
Script.update.disconnect(dataCaptureUpdateLoop);
|
|
updateConnected = false;
|
|
}
|
|
sendToQml({ method: 'updateIsRecordingSelectedGestures', isRecordingSelectedGestures: isRecordingSelectedGestures });
|
|
Audio.playSound(SOUND_GESTURE_RECORDING_STOP, {
|
|
position: MyAvatar.position,
|
|
localOnly: true,
|
|
volume: 0.8
|
|
});
|
|
|
|
var timestamp;
|
|
var index = -1;
|
|
|
|
if (jointDataToRecord[0]) {
|
|
var dataToSave = {
|
|
timestamp: false,
|
|
data: []
|
|
};
|
|
leftHandRecorder.stopRecording();
|
|
timestamp = leftHandRecorder.recordingStartTimeMS;
|
|
dataToSave.timestamp = timestamp;
|
|
dataToSave.data = leftHandRecorder.currentPoseFrameData;
|
|
leftHandRecorder.previouslyRecordedFrameData.push(dataToSave);
|
|
leftHandRecorder.currentPoseFrameData = [];
|
|
index = leftHandRecorder.previouslyRecordedFrameData.length - 1;
|
|
} else {
|
|
leftHandRecorder.previouslyRecordedFrameData.push("NODATA");
|
|
}
|
|
|
|
if (jointDataToRecord[1]) {
|
|
var dataToSave = {
|
|
timestamp: false,
|
|
data: []
|
|
};
|
|
rightHandRecorder.stopRecording();
|
|
timestamp = rightHandRecorder.recordingStartTimeMS;
|
|
dataToSave.timestamp = timestamp;
|
|
dataToSave.data = rightHandRecorder.currentPoseFrameData;
|
|
rightHandRecorder.previouslyRecordedFrameData.push(dataToSave);
|
|
rightHandRecorder.currentPoseFrameData = [];
|
|
index = rightHandRecorder.previouslyRecordedFrameData.length - 1;
|
|
} else {
|
|
rightHandRecorder.previouslyRecordedFrameData.push("NODATA");
|
|
}
|
|
|
|
if (jointDataToRecord[2]) {
|
|
var dataToSave = {
|
|
timestamp: false,
|
|
data: []
|
|
};
|
|
headRecorder.stopRecording();
|
|
timestamp = headRecorder.recordingStartTimeMS;
|
|
dataToSave.timestamp = timestamp;
|
|
dataToSave.data = headRecorder.currentPoseFrameData;
|
|
headRecorder.previouslyRecordedFrameData.push(dataToSave);
|
|
headRecorder.currentPoseFrameData = [];
|
|
index = headRecorder.previouslyRecordedFrameData.length - 1;
|
|
} else {
|
|
headRecorder.previouslyRecordedFrameData.push("NODATA");
|
|
}
|
|
|
|
timestamp = new Date(timestamp);
|
|
|
|
sendToQml({
|
|
method: 'appendRecordedGesture',
|
|
index: index,
|
|
recordedTime: timestamp.getHours() + ":" + timestamp.getMinutes() + ":" + timestamp.getSeconds(),
|
|
recordedJoints: [jointDataToRecord[0], jointDataToRecord[1], jointDataToRecord[2]]
|
|
});
|
|
|
|
updateGestureDetectionSystem();
|
|
}
|
|
|
|
var isRecordingSelectedGestures = false;
|
|
function toggleRecordingGesture() {
|
|
if (!isRecordingSelectedGestures) {
|
|
startRecordingSelectedGestures();
|
|
} else {
|
|
stopRecordingSelectedGestures();
|
|
}
|
|
}
|
|
/********************************
|
|
// END Global Capture Start/Stop/Update functions
|
|
********************************/
|
|
|
|
/********************************
|
|
// START Controller Mapping
|
|
********************************/
|
|
var gesturesControllerMapping = false;
|
|
var gesturesControllerMappingName = 'Hifi-Gestures-Mapping';
|
|
function maybeRegisterButtonMappings() {
|
|
// Don't re-register
|
|
if (gesturesControllerMapping) {
|
|
return;
|
|
}
|
|
gesturesControllerMapping = Controller.newMapping(gesturesControllerMappingName);
|
|
if (controllerType === "OculusTouch") {
|
|
gesturesControllerMapping.from(Controller.Standard.RS).to(function (value) {
|
|
if (value === 1.0) {
|
|
toggleRecordingGesture();
|
|
}
|
|
return;
|
|
});
|
|
} else if (controllerType === "Vive") {
|
|
gesturesControllerMapping.from(Controller.Standard.RightPrimaryThumb).to(function (value) {
|
|
if (value === 1.0) {
|
|
toggleRecordingGesture();
|
|
}
|
|
return;
|
|
});
|
|
}
|
|
gesturesControllerMapping.enable();
|
|
}
|
|
|
|
function disableButtonMappings() {
|
|
if (gesturesControllerMapping) {
|
|
gesturesControllerMapping.disable();
|
|
gesturesControllerMapping = false;
|
|
}
|
|
}
|
|
/********************************
|
|
// END Controller Mapping
|
|
********************************/
|
|
|
|
/********************************
|
|
// START App-Related Functions
|
|
********************************/
|
|
// Function Name: sendToQml()
|
|
//
|
|
// Description:
|
|
// -Use this function to send a message to the app's QML (i.e. to change appearances). The "message" argument is what is sent to
|
|
// the app's QML in the format "{method, params}", like json-rpc. See also fromQml().
|
|
function sendToQml(message) {
|
|
ui.sendMessage(message);
|
|
}
|
|
|
|
// Function Name: fromQml()
|
|
//
|
|
// Description:
|
|
// -Called when a message is received from the app QML. The "message" argument is what is sent from the app QML
|
|
// in the format "{method, params}", like json-rpc. See also sendToQml().
|
|
function fromQml(message) {
|
|
switch (message.method) {
|
|
case 'enableRecordingSwitchChanged':
|
|
if (message.status) {
|
|
maybeRegisterButtonMappings();
|
|
} else {
|
|
disableButtonMappings();
|
|
}
|
|
Settings.setValue('gestures/enableControllerMapping', message.status);
|
|
break;
|
|
case 'updateJointSelections':
|
|
jointDataToRecord = message.selections;
|
|
Settings.setValue('gestures/captureLeftHandData', jointDataToRecord[0]);
|
|
Settings.setValue('gestures/captureRightHandData', jointDataToRecord[1]);
|
|
Settings.setValue('gestures/captureHeadData', jointDataToRecord[2]);
|
|
break;
|
|
case 'toggleManualDataCapture':
|
|
toggleRecordingGesture();
|
|
break;
|
|
case 'modifyListeningForGestureIndex':
|
|
gestureToDetect_index = message.listeningForGestureIndex;
|
|
updateGestureDetectionSystem();
|
|
break;
|
|
case 'clearRecordedGestures':
|
|
gestureToDetect_index = -1;
|
|
leftHandRecorder.previouslyRecordedFrameData = [];
|
|
rightHandRecorder.previouslyRecordedFrameData = [];
|
|
headRecorder.previouslyRecordedFrameData = [];
|
|
break;
|
|
case 'updateSensitivity':
|
|
gestureDetectionSensitivityMultiplier = message.sensitivity;
|
|
Settings.setValue('gestures/sensitivity', gestureDetectionSensitivityMultiplier);
|
|
break;
|
|
case 'updateMaxIgnoreDistance':
|
|
avatarIgnoreMaxDistance = message.distance;
|
|
Settings.setValue('gestures/maxIgnoreDistance', avatarIgnoreMaxDistance);
|
|
maybePrint("ZRF: Updating max avatar ignore distance to '" + avatarIgnoreMaxDistance, DEBUG_IMPORTANT);
|
|
break;
|
|
case 'copyDataToClipboard':
|
|
var dataToCopy = [];
|
|
var leftHandData = "NODATA";
|
|
var rightHandData = "NODATA";
|
|
var headData = "NODATA";
|
|
|
|
if (leftHandRecorder.previouslyRecordedFrameData[message.index] !== "NODATA") {
|
|
leftHandData = leftHandRecorder.previouslyRecordedFrameData[message.index];
|
|
}
|
|
if (rightHandRecorder.previouslyRecordedFrameData[message.index] !== "NODATA") {
|
|
rightHandData = rightHandRecorder.previouslyRecordedFrameData[message.index];
|
|
}
|
|
if (headRecorder.previouslyRecordedFrameData[message.index] !== "NODATA") {
|
|
headData = headRecorder.previouslyRecordedFrameData[message.index];
|
|
}
|
|
|
|
dataToCopy.push(leftHandData);
|
|
dataToCopy.push(rightHandData);
|
|
dataToCopy.push(headData);
|
|
|
|
Window.copyToClipboard(JSON.stringify(dataToCopy));
|
|
break;
|
|
case 'importGestureData':
|
|
var data = JSON.parse(message.data);
|
|
if (data.length !== 3) {
|
|
maybePrint('Unrecognized import data format, bailing!', DEBUG_URGENT);
|
|
return;
|
|
}
|
|
|
|
leftHandRecorder.previouslyRecordedFrameData.push(data[0]);
|
|
rightHandRecorder.previouslyRecordedFrameData.push(data[1]);
|
|
headRecorder.previouslyRecordedFrameData.push(data[2]);
|
|
|
|
appendGestureToQMLWithIndex(leftHandRecorder.previouslyRecordedFrameData.length - 1);
|
|
break;
|
|
default:
|
|
maybePrint('Unrecognized message from Gestures.qml: ' + JSON.stringify(message), DEBUG_URGENT);
|
|
}
|
|
}
|
|
|
|
function appendGestureToQMLWithIndex(index) {
|
|
var timestamp;
|
|
var recordedJoints = [false, false, false];
|
|
|
|
if (leftHandRecorder.previouslyRecordedFrameData[index] !== "NODATA") {
|
|
recordedJoints[0] = true;
|
|
timestamp = leftHandRecorder.previouslyRecordedFrameData[index].timestamp;
|
|
}
|
|
if (rightHandRecorder.previouslyRecordedFrameData[index] !== "NODATA") {
|
|
recordedJoints[1] = true;
|
|
timestamp = rightHandRecorder.previouslyRecordedFrameData[index].timestamp;
|
|
}
|
|
if (headRecorder.previouslyRecordedFrameData[index] !== "NODATA") {
|
|
recordedJoints[2] = true;
|
|
timestamp = headRecorder.previouslyRecordedFrameData[index].timestamp;
|
|
}
|
|
|
|
timestamp = new Date(timestamp);
|
|
|
|
sendToQml({
|
|
method: 'appendRecordedGesture',
|
|
index: index,
|
|
recordedTime: timestamp.getHours() + ":" + timestamp.getMinutes() + ":" + timestamp.getSeconds(),
|
|
recordedJoints: recordedJoints
|
|
});
|
|
}
|
|
|
|
function appendAllGesturesToQML() {
|
|
for (var i = 0; i < leftHandRecorder.previouslyRecordedFrameData.length; i++) {
|
|
appendGestureToQMLWithIndex(i);
|
|
}
|
|
}
|
|
|
|
// Function Name: appUiOpened()
|
|
//
|
|
// Description:
|
|
// - Called when the app's UI is opened
|
|
//
|
|
var APP_INITIALIZE_UI_DELAY = 500; // MS
|
|
function appUiOpened() {
|
|
// In the case of a remote QML app, it takes a bit of time
|
|
// for the event bridge to actually connect, so we have to wait...
|
|
Script.setTimeout(function () {
|
|
sendToQml({
|
|
method: 'initializeUI',
|
|
masterSwitchOn: !!gesturesControllerMapping,
|
|
jointDataToRecord: jointDataToRecord,
|
|
currentlyDetectingGesture: gestureToDetect_index,
|
|
gestureDetectionSensitivityMultiplier: gestureDetectionSensitivityMultiplier,
|
|
maxIgnoreDistance: avatarIgnoreMaxDistance
|
|
});
|
|
|
|
appendAllGesturesToQML();
|
|
}, APP_INITIALIZE_UI_DELAY);
|
|
}
|
|
|
|
// Function Name: appUiClosed()
|
|
//
|
|
// Description:
|
|
// - Called when the app's UI is closed
|
|
//
|
|
function appUiClosed() {
|
|
}
|
|
|
|
// Function Name: startup()
|
|
//
|
|
// Description:
|
|
// -startup() will be called when the script is loaded.
|
|
//
|
|
var ui;
|
|
var controllerType = "Other";
|
|
var avatarIgnoreMaxDistance = 5.0;
|
|
function startup() {
|
|
ui = new AppUi({
|
|
buttonName: "GESTURES",
|
|
home: Script.resolvePath('./Gestures.qml'),
|
|
onOpened: appUiOpened,
|
|
onClosed: appUiClosed,
|
|
onMessage: fromQml,
|
|
sortOrder: 15,
|
|
normalButton: Script.resourcesPath() + "icons/tablet-icons/avatar-record-i.svg",
|
|
activeButton: Script.resourcesPath() + "icons/tablet-icons/avatar-record-a.svg"
|
|
});
|
|
|
|
// Controller type detection
|
|
var VRDevices = Controller.getDeviceNames().toString();
|
|
if (VRDevices) {
|
|
if (VRDevices.indexOf("Vive") !== -1) {
|
|
controllerType = "Vive";
|
|
} else if (VRDevices.indexOf("OculusTouch") !== -1) {
|
|
controllerType = "OculusTouch";
|
|
}
|
|
}
|
|
|
|
if (Settings.getValue('gestures/enableControllerMapping', false)) {
|
|
maybeRegisterButtonMappings();
|
|
}
|
|
|
|
avatarIgnoreMaxDistance = Settings.getValue('gestures/maxIgnoreDistance', 5.0);
|
|
}
|
|
|
|
// Function Name: shutdown()
|
|
//
|
|
// Description:
|
|
// - Called when the script ends (i.e. is stopped).
|
|
//
|
|
function shutdown() {
|
|
appUiClosed();
|
|
}
|
|
|
|
var SOUND_GESTURE_RECORDING_START = SoundCache.getSound(Script.resolvePath("startRecording.wav"));
|
|
var SOUND_GESTURE_RECORDING_STOP = SoundCache.getSound(Script.resolvePath("stopRecording.wav"));
|
|
var SOUND_GESTURE_DETECTED = SoundCache.getSound(Script.resolvePath("gestureDetected.wav"));
|
|
startup();
|
|
Script.scriptEnding.connect(shutdown);
|
|
/********************************
|
|
// END App-Related Functions
|
|
********************************/
|
|
|
|
}()); // END LOCAL_SCOPE
|