"use strict";
/*jslint vars:true, plusplus:true, forin:true*/
/*global Tablet, Script, Users, console */
/* eslint indent: ["error", 4, { "outerIIFEBody": 0 }] */
//
// gestures.js
//
// Created by Zach Fox on 2018-07-24
// Copyright 2018 High Fidelity, Inc
//
// Distributed under the Apache License, Version 2.0
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
(function () { // BEGIN LOCAL_SCOPE
var AppUi = Script.require('appUi');
/********************************
// START Debug Functions
********************************/
var DEBUG_UNIMPORTANT = 0;
var DEBUG_IMPORTANT = 1;
var DEBUG_URGENT = 2;
var DEBUG_ONLY_PRINT_URGENT = 0;
var DEBUG_PRINT_URGENT_AND_IMPORTANT = 1;
var DEBUG_PRINT_EVERYTHING = 2;
var debugLevel = DEBUG_ONLY_PRINT_URGENT;
var avatarIgnoreID = null;
var borderEntityID = null;
var loadingBarEntityID = null;
var isBlockFocus = true;
var blockFurtherDetection = false;
function maybePrint(string, importance) {
if (importance >= (DEBUG_URGENT - debugLevel)) {
console.log(string);
}
}
/********************************
// END Debug Functions
********************************/
/********************************
//overlay code
*******************************/
var ROOT = "http://mpassets.highfidelity.com/62f401b8-cc95-4e18-8723-12f076e6e5fd-v1/";
var startFrame = 1;
var inFrontOfMyAvatar = getPositionForIcon();
inFrontOfMyAvatar.x = inFrontOfMyAvatar.x - 0.51;
var animation = {
setupAnimation: function (url, width, height, maxFrames, fps, text) {
maxFrames = maxFrames || 1;
var anim = {
timer: null,
url: url,
frames: {},
running: false,
fps: 1000 / fps,
currentFrame: startFrame,
maxFrames: maxFrames,
width: width,
height: height,
emissive: true,
text: text,
blockedYet: false
};
for (var i = startFrame; i <= maxFrames; i++) {
anim.frames[i] = TextureCache.prefetch(url.slice(0, -4) + i + ".png");
}
anim.border = Overlays.addOverlay("image3d", {
position: getPositionForIcon(),
drawInFront: true,
isFacingAvatar: true,
url: "http://hifi-content.s3.amazonaws.com/angus/gesture/outline.png",
alpha: 0.0,
ignorePickIntersection: true,
parentID: MyAvatar.sessionUUID
});
anim.loadBar = Overlays.addOverlay("image3d", {
parentID: anim.border,
localPosition: {x: -0.48, y: 0, z: 0}, // getPositionForIcon(),
dimensions: {x: 0.0, y: 0.5},
drawInFront: false,
isFacingAvatar: true,
url: "http://hifi-content.s3.amazonaws.com/angus/gesture/loadingBar.png",
alpha: 0.0,
ignorePickIntersection: true
});
anim.uuid = Overlays.addOverlay("image3d", {
position: getPositionForIcon(),
drawInFront: true,
dimensions: {x: 0.5, y: 0.5},
isFacingAvatar: true,
url: anim.frames[startFrame].url,
alpha: 0,
ignorePickIntersection: true,
parentID: MyAvatar.sessionUUID
});
anim.textUUID = Overlays.addOverlay("text3d", {
localPosition: {x: -(anim.text.length * 0.00875), y: 0.0, z: 0},
parentID: anim.uuid,
drawInFront: true,
dimensions: {x: 0.0, y: 0.0},
leftMargin: -0.04,
topMargin: -0.01,
isFacingAvatar: true,
text: anim.text,
lineHeight: 0.07,
textAlpha: 0,
alpha: 0,
ignorePickIntersection: true,
fontsize: 36
});
anim.end = function (stopped) {
Script.clearInterval(anim.timer);
anim.running = false;
anim.currentFrame = startFrame;
Overlays.editOverlay(anim.uuid, {alpha: 0});
Overlays.editOverlay(anim.textUUID, {textAlpha: 0});
Overlays.editOverlay(anim.border, {alpha: 0});
Overlays.editOverlay(anim.loadBar, {alpha: 0});
//if (!stopped) {
// anim.callback(anim.callbackData);
//}
anim.blockedYet = false;
};
anim.stop = function () {
anim.end(true);
};
anim.nextFrame = function () {
anim.currentFrame++;
if (anim.currentFrame >= anim.maxFrames) {
if (anim.running) {
anim.end(false);
}
}
if (anim.currentFrame < (anim.maxFrames - 20)) {
Overlays.editOverlay(anim.loadBar, {
localPosition: { x: - (0.47 - (anim.currentFrame/(anim.maxFrames-21))*0.47), y: 0, z: 0 },
dimensions: {x: (anim.currentFrame/(anim.maxFrames-21))*0.97, y: 0.48},
});
} else {
Overlays.editOverlay(anim.textUUID, {
text: "USER BLOCKED"
});
if (!anim.blockedYet) {
anim.blockedYet = true;
anim.callback(anim.callbackData);
}
}
var propertiesToGet = {};
propertiesToGet[anim.uuid] = ["rotation","parentID"];
var props = Overlays.getOverlaysProperties(propertiesToGet);
};
anim.start = function (position, _callback, _callbackData) {
if (!anim.running) {
anim.running = true;
anim.callback = _callback;
anim.callbackData = _callbackData;
Overlays.editOverlay(anim.uuid, {
url: anim.frames[startFrame].url,
position: position,
alpha: 0
});
Overlays.editOverlay(anim.textUUID, {
textAlpha: 1,
text: anim.text,
});
Overlays.editOverlay(anim.border, {
position: position,
alpha: 1
});
Overlays.editOverlay(anim.loadBar, {
localPosition: {x: -0.47, y: 0, z: 0},
alpha: 1
});
if (anim.timer) {
Script.clearInterval(anim.timer);
}
anim.timer = Script.setInterval(anim.nextFrame, anim.fps);
}
};
return anim;
}
};
var animShush = animation.setupAnimation(ROOT + "loading/loading.png", 200, 200, 60, 30, "Hold to Block");
function getPositionForIcon() {
var camPos = Camera.position;
var camRot = Camera.orientation;
return Vec3.sum(camPos, Vec3.multiplyQbyV(camRot, {x: 0, y: 0, z: -1}));
}
function shushPerson(hand) {
if ((avatarIgnoreID !== null) && !isBlockFocus) {
Users.ignore(avatarIgnoreID);
avatarIgnoreID = null;
isBlockFocus = true;
}
blockFurtherDetection = false;
print("sush timer expired");
}
/********************************
// START GestureRecorder
********************************/
var LEFT_HAND_JOINT_NAME = "leftHand";
var RIGHT_HAND_JOINT_NAME = "rightHand";
var HEAD_JOINT_NAME = "head";
function GestureRecorder(jointName) {
this.jointName = jointName;
this.currentPoseFrameData = [];
this.previouslyRecordedFrameData = [];
this.recordingStartTimeMS;
this.recordingLengthSEC;
this.isRecordingGesture = false;
}
GestureRecorder.prototype.initializeDataForRecording = function () {
this.currentPoseFrameData = [];
this.recordingStartTimeMS = Date.now();
maybePrint("ZRF: Starting gesture recording for joint '" + this.jointName + "'.", DEBUG_IMPORTANT);
};
var Y_AXIS = { x: 0, y: 1, z: 0 };
GestureRecorder.prototype.sampleFrame = function (timeSinceStartSEC, previousSample) {
var pose;
var headPose = Controller.getPoseValue(Controller.Standard.Head);
if (this.jointName === LEFT_HAND_JOINT_NAME) {
pose = Controller.getPoseValue(Controller.Standard.LeftHand);
} else if (this.jointName === RIGHT_HAND_JOINT_NAME) {
pose = Controller.getPoseValue(Controller.Standard.RightHand);
pose.rotation = Quat.multiply(Quat.inverse(headPose.rotation), pose.rotation);
var handOffset = Vec3.subtract(pose.translation, headPose.translation);
pose.translation = Vec3.multiplyQbyV(Quat.inverse(headPose.rotation), handOffset);
var headDotHand = Vec3.dot({x: 0.0,y: 0.0,z: 1.0}, Vec3.normalize(pose.translation));
if (headDotHand < 0.866) {
isBlockFocus = true;
if (animShush.running) {
animShush.stop();
}
blockFurtherDetection = false;
}
// print("isBlockFocus " + isBlockFocus);
} else if (this.jointName === HEAD_JOINT_NAME) {
pose = Controller.getPoseValue(Controller.Standard.Head);
// pose.rotation = Quat.fromVec3Degrees({ x: 0.0, y: 0.0, z: 0.0 });
// pose.translation = { x: 0.0, y: 0.0, z: 0.0 };
}
// make things relative to the head
// pose.rotation = Quat.multiply(Quat.inverse(headPose.rotation), pose.rotation);
// pose.translation = Vec3.subtract(pose.translation, headPose.translation);
var frameData = {
timeSinceStartSEC: timeSinceStartSEC,
x: pose.translation.x,
y: pose.translation.y,
z: pose.translation.z,
rotation: pose.rotation
};
return frameData;
};
var MS_PER_SEC = 1000;
GestureRecorder.prototype.captureDataNow = function () {
var now = Date.now();
var timeSinceStartSEC = (now - this.recordingStartTimeMS) / MS_PER_SEC;
this.currentPoseFrameData.push(this.sampleFrame(timeSinceStartSEC));
};
GestureRecorder.prototype.stopRecording = function () {
this.recordingLengthSEC = (Date.now() - this.recordingStartTimeMS) / MS_PER_SEC;
calculateDerivatives(this.currentPoseFrameData);
maybePrint("ZRF: Finished gesture recording for joint '" + this.jointName + "'.", DEBUG_IMPORTANT);
JSON.stringify(this.currentPoseFrameData, null, 4).split("\n").forEach(function (str) {
maybePrint(str, DEBUG_UNIMPORTANT);
});
};
GestureRecorder.prototype.gestureDetectCheck = function () {
maybePrint("gestureDetect() currentPoseFrameData.length = " + this.currentPoseFrameData.length +
", previouslyRecordedFrameData.data.length = " + this.previouslyRecordedFrameData[gestureToDetect_index].data.length +
", recordingLength = " + this.recordingLengthSEC + " (sec)", DEBUG_UNIMPORTANT);
// not enough frames to test
if (this.currentPoseFrameData.length < this.previouslyRecordedFrameData[gestureToDetect_index].data.length / 2) {
return false;
}
var i = 0, j = 0;
var framesTested = 0;
var framesPassed = 0;
while (i < this.previouslyRecordedFrameData[gestureToDetect_index].data.length && j < this.currentPoseFrameData.length) {
var it = this.previouslyRecordedFrameData[gestureToDetect_index].data[i].timeSinceStartSEC;
var jt = this.currentPoseFrameData[j].timeSinceStartSEC - this.currentPoseFrameData[0].timeSinceStartSEC;
if (jt < it) {
var iPrev = i > 0 ? (i - 1) : 0;
var a = this.previouslyRecordedFrameData[gestureToDetect_index].data[iPrev];
var b = this.previouslyRecordedFrameData[gestureToDetect_index].data[i];
var alpha = i > 0 ? (jt - a.timeSinceStartSEC) / (b.timeSinceStartSEC - a.timeSinceStartSEC) : 0;
var frame = lerpFrame(a, b, alpha);
framesTested++;
if (testFrames(this.currentPoseFrameData[j], frame)) {
framesPassed++;
}
j++;
} else {
i++;
}
}
// console.log("frames tested and passed " + framesTested + " " + framesPassed );
if (framesPassed / framesTested > 0.4) {
maybePrint(~~((framesPassed / framesTested) * 100) + "% match", DEBUG_UNIMPORTANT);
}
return framesPassed / framesTested > 0.75;
};
GestureRecorder.prototype.gestureDetect = function () {
var t = Date.now() / MS_PER_SEC;
this.currentPoseFrameData.push(this.sampleFrame(t));
calculateDerivatives(this.currentPoseFrameData);
var dataLength = this.previouslyRecordedFrameData[gestureToDetect_index].data.length;
var i;
for (i = 0; i < this.currentPoseFrameData.length; i++) {
if (i > 0 && this.currentPoseFrameData[i].timeSinceStartSEC > t - this.previouslyRecordedFrameData[gestureToDetect_index].data[dataLength-1].timeSinceStartSEC) {
this.currentPoseFrameData = this.currentPoseFrameData.slice(i - 1);
break;
}
}
// console.log("length of recording " + this.previouslyRecordedFrameData[gestureToDetect_index].data[dataLength-1].timeSinceStartSEC + " length of currentData " + this.currentPoseFrameData.length);
if (this.currentPoseFrameData.length > 0 && this.gestureDetectCheck()) {
maybePrint("ZRF: Gesture with index " + gestureToDetect_index + " detected on joint " + this.jointName + "!", DEBUG_UNIMPORTANT);
return true;
}
return false;
};
/********************************
// END GestureRecorder
********************************/
/********************************
// START Shared Math Utility Functions
********************************/
// Function Name: inFrontOf()
//
// Description:
// -Returns the position in front of the given "position" argument, where the forward vector is based off
// the "orientation" argument and the amount in front is based off the "distance" argument.
function inFrontOf(distance, position, orientation) {
return Vec3.sum(position || MyAvatar.position,
Vec3.multiply(distance, Quat.getForward(orientation || MyAvatar.orientation)));
}
function calculateDerivatives(frames) {
var i, length = frames.length;
var keys = ["x", "y", "z"];
var dKeys = ["dx", "dy", "dz"];
for (i = 0; i < length; i++) {
var prevIndex = (i === 0) ? 0 : i - 1;
var nextIndex = (i === length - 1) ? i : i + 1;
var j = 0, numKeys = keys.length;
for (j = 0; j < numKeys; j++) {
var d1 = frames[i][keys[j]] - frames[prevIndex][keys[j]];
var d2 = frames[nextIndex][keys[j]] - frames[i][keys[j]];
frames[i][dKeys[j]] = (d1 + d2) / 2;
}
}
}
function lerp(a, b, alpha) {
return a * (1 - alpha) + b * alpha;
}
function lerpFrame(a, b, alpha) {
var keys = Object.keys(a);
var result = {};
keys.forEach(function (key) {
result[key] = lerp(a[key], b[key], alpha);
});
return result;
}
var RAD_TO_DEG = 180 / Math.PI;
var DEG_TO_RAD = Math.PI / 180;
var ROTATION_THRESHOLD = 30.0 * DEG_TO_RAD; // radians
var X_THRESHOLD = 0.2; // meters
var Y_THRESHOLD = 0.2; // meters
var Z_THRESHOLD = 0.2; // meters
var DX_THRESHOLD = 0.02; // meters / sec
var DY_THRESHOLD = 0.02; // meters / sec
var DZ_THRESHOLD = 0.02; // meters / sec
function testFrames(a, b) {
//console.log("gesture test frames");
if (Math.abs(Quat.dot(a.rotation, b.rotation)) < Math.cos(ROTATION_THRESHOLD * gestureDetectionSensitivityMultiplier)) {
return false;
}
if (Math.abs(a.x - b.x) > X_THRESHOLD * gestureDetectionSensitivityMultiplier) {
return false;
}
if (Math.abs(a.y - b.y) > Y_THRESHOLD * gestureDetectionSensitivityMultiplier) {
return false;
}
if (Math.abs(a.z - b.z) > Z_THRESHOLD * gestureDetectionSensitivityMultiplier) {
return false;
}
if (Math.abs(a.dx - b.dx) > DX_THRESHOLD * gestureDetectionSensitivityMultiplier) {
return false;
}
if (Math.abs(a.dy - b.dy) > DY_THRESHOLD * gestureDetectionSensitivityMultiplier) {
return false;
}
if (Math.abs(a.dz - b.dz) > DZ_THRESHOLD * gestureDetectionSensitivityMultiplier) {
return false;
}
return true;
}
/********************************
// END Shared Math Utility Functions
********************************/
/********************************
// START Global Detection Start/Stop/Update functions
********************************/
var isDetectingGesture = false;
var gestureToDetect_index = -1;
var gestureDetectionSensitivityMultiplier = Settings.getValue('gestures/sensitivity', 1.0);
function updateGestureDetectionSystem() {
if (gestureToDetect_index > -1 && !isRecordingSelectedGestures) {
if (isDetectingGesture) {
Script.update.disconnect(gestureDetectionUpdateLoop);
isDetectingGesture = false;
}
isDetectingGesture = true;
Script.update.connect(gestureDetectionUpdateLoop);
} else {
if (isDetectingGesture) {
Script.update.disconnect(gestureDetectionUpdateLoop);
isDetectingGesture = false;
leftHandRecorder.currentPoseFrameData = [];
rightHandRecorder.currentPoseFrameData = [];
headRecorder.currentPoseFrameData = [];
}
}
maybePrint("ZRF: The gesture to detect has index: " + gestureToDetect_index + ". Currently detecting gestures: " + isDetectingGesture, DEBUG_IMPORTANT);
}
var detectedEntity = false;
var deleteDetectedEntityTimeout = false;
function handleDetectedEntity() {
if (!detectedEntity) {
detectedEntity = Entities.addEntity({
"collidesWith": "",
"collisionMask": 0,
"collisionless": true,
"color": {
"blue": 20,
"green": 200,
"red": 20
},
"dimensions": {
"blue": 0.05000000074505806,
"green": 0.4000000059604645,
"red": 0.4000000059604645,
"x": 0.4000000059604645,
"y": 0.4000000059604645,
"z": 0.05000000074505806
},
"ignoreForCollisions": true,
"shape": "Cube",
"type": "Box",
"userData": "{\"grabbableKey\":{\"grabbable\":false}}",
"position": inFrontOf(0.8, Camera.position, Camera.orientation),
"rotation": Camera.orientation
}, true);
}
if (deleteDetectedEntityTimeout) {
Script.clearTimeout(deleteDetectedEntityTimeout);
}
deleteDetectedEntityTimeout = Script.setTimeout(function () {
if (detectedEntity) {
Entities.deleteEntity(detectedEntity);
detectedEntity = false;
}
deleteDetectedEntityTimeout = false;
}, 1000);
}
function handleEntityPickRay() {
var pickRay = {
origin: Camera.position,
direction: Quat.getFront(Camera.orientation),
length: 100
};
var entityIntersection = Entities.findRayIntersection(pickRay, true);
if (entityIntersection.intersects) {
var intersectEntityID = entityIntersection.entityID;
Entities.editEntity(intersectEntityID, {
color: {
red: Math.random() * 255,
green: Math.random() * 255,
blue: Math.random() * 255
}
});
}
}
function handleAvatarPickRay() {
var myPos = Camera.position;
var myPos1 = {x: Camera.position.x, y: (Camera.position.y - 1.3), z: Camera.position.z};
var myPos2 = {x: Camera.position.x, y: (Camera.position.y - 1.1), z: Camera.position.z};
var myPos3 = {x: Camera.position.x, y: (Camera.position.y - 0.9), z: Camera.position.z};
var myPos4 = {x: Camera.position.x, y: (Camera.position.y - 0.7), z: Camera.position.z};
var myPos5 = {x: Camera.position.x, y: (Camera.position.y - 0.5), z: Camera.position.z};
var myPos6 = {x: Camera.position.x, y: (Camera.position.y - 0.4), z: Camera.position.z};
var forwardDirection = Quat.getFront(Camera.orientation);
forwardDirection.y = 0.0;
var flatForward = Vec3.normalize(forwardDirection);
var offsetUp = Vec3.sum(Vec3.multiply(0.1, Quat.getUp(Camera.orientation)),myPos);
var forwardEnd = Vec3.multiply(5.0,Quat.getFront(Camera.orientation));
var pickRay = {
origin: Camera.position,
direction: Quat.getFront(Camera.orientation),
length: avatarIgnoreMaxDistance
};
var pickRay1 = {
origin: myPos1,
direction: flatForward,
length: avatarIgnoreMaxDistance
};
var pickRay2 = {
origin: myPos2,
direction: flatForward,
length: avatarIgnoreMaxDistance
};
var pickRay3 = {
origin: myPos3,
direction: flatForward,
length: avatarIgnoreMaxDistance
};
var pickRay4 = {
origin: myPos4,
direction: flatForward,
length: avatarIgnoreMaxDistance
};
var pickRay5 = {
origin: myPos5,
direction: flatForward,
length: avatarIgnoreMaxDistance
};
var pickRay6 = {
origin: myPos2,
direction: flatForward,
length: avatarIgnoreMaxDistance
};
var avatarIntersection = AvatarList.findRayIntersection(pickRay, [], [MyAvatar.sessionUUID]);
var avatarIntersection1 = AvatarList.findRayIntersection(pickRay1, [], [MyAvatar.sessionUUID]);
var avatarIntersection2 = AvatarList.findRayIntersection(pickRay2, [], [MyAvatar.sessionUUID]);
var avatarIntersection3 = AvatarList.findRayIntersection(pickRay3, [], [MyAvatar.sessionUUID]);
var avatarIntersection4 = AvatarList.findRayIntersection(pickRay4, [], [MyAvatar.sessionUUID]);
var avatarIntersection5 = AvatarList.findRayIntersection(pickRay5, [], [MyAvatar.sessionUUID]);
var avatarIntersection6 = AvatarList.findRayIntersection(pickRay6, [], [MyAvatar.sessionUUID]);
var ret = false;
// console.log("camera position" + JSON.stringify(Camera.position) );
if (avatarIntersection.intersects) {
avatarIgnoreID = avatarIntersection.avatarID;
ret = true;
console.log("we have a hit");
} else if (avatarIntersection1.intersects) {
avatarIgnoreID = avatarIntersection1.avatarID;
ret = true;
console.log("we have a hit--1");
} else if (avatarIntersection2.intersects) {
avatarIgnoreID = avatarIntersection2.avatarID;
ret = true;
console.log("we have a hit----2");
} else if (avatarIntersection3.intersects) {
avatarIgnoreID = avatarIntersection3.avatarID;
ret = true;
console.log("we have a hit------3");
} else if (avatarIntersection4.intersects) {
avatarIgnoreID = avatarIntersection4.avatarID;
ret = true;
console.log("we have a hit--------4");
} else if (avatarIntersection5.intersects) {
avatarIgnoreID = avatarIntersection5.avatarID;
ret = true;
console.log("we have a hit----------5");
} else if (avatarIntersection6.intersects) {
avatarIgnoreID = avatarIntersection6.avatarID;
ret = true;
console.log("we have a hit------------6");
}
return ret;
}
function performGestureDetectedAction() {
//maybePrint("ZRF: GESTURE WITH INDEX " + gestureToDetect_index + " DETECTED!", DEBUG_URGENT);
sendToQml({ method: 'gestureDetected' });
//handleDetectedEntity();
handleEntityPickRay();
var hitAvatar = handleAvatarPickRay();
//
if (hitAvatar) {
console.log("did we pick an avatar? " + hitAvatar);
//Audio.playSound(SOUND_GESTURE_DETECTED, {
// position: MyAvatar.position,
// localOnly: true,
// volume: 0.3
//});
isBlockFocus = false;
animShush.start(getPositionForIcon(), shushPerson, 0);
} else {
blockFurtherDetection = false;
}
// print("avatar is in my view " + hitAvatar);
}
function gestureDetectionUpdateLoop() {
var detected = [false, false, false];
var mustBeDetected = [
leftHandRecorder.previouslyRecordedFrameData[gestureToDetect_index] !== "NODATA",
rightHandRecorder.previouslyRecordedFrameData[gestureToDetect_index] !== "NODATA",
headRecorder.previouslyRecordedFrameData[gestureToDetect_index] !== "NODATA"
];
if (mustBeDetected[0]) {
detected[0] = leftHandRecorder.gestureDetect();
}
if (mustBeDetected[1]) {
detected[1] = rightHandRecorder.gestureDetect();
}
if (mustBeDetected[2]) {
detected[2] = headRecorder.gestureDetect();
}
var allNecessaryGesturesDetected = false;
for (var i = 0; i < 3; i++) {
if (detected[i] !== mustBeDetected[i]) {
allNecessaryGesturesDetected = false;
break;
} else {
allNecessaryGesturesDetected = true;
}
}
//var GREEN = { r: 0, g: 1, b: 0, a: 1 };
//var lookingRay = Vec3.multiply(5.0, Quat.getForward(Camera.orientation));
//DebugDraw.drawRay(Camera.position, Vec3.sum(Camera.position,lookingRay), GREEN);
// console.log(" block further detection " + allNecessaryGesturesDetected + " " + blockFurtherDetection);
if (allNecessaryGesturesDetected && !blockFurtherDetection) {
blockFurtherDetection = true;
performGestureDetectedAction();
leftHandRecorder.currentPoseFrameData = [];
rightHandRecorder.currentPoseFrameData = [];
headRecorder.currentPoseFrameData = [];
}
var myPos = Camera.position;
var myPos1 = {x: Camera.position.x, y: (Camera.position.y - 1.3), z: Camera.position.z};
var myPos2 = {x: Camera.position.x, y: (Camera.position.y - 1.1), z: Camera.position.z};
var myPos3 = {x: Camera.position.x, y: (Camera.position.y - 0.9), z: Camera.position.z};
var myPos4 = {x: Camera.position.x, y: (Camera.position.y - 0.7), z: Camera.position.z};
var myPos5 = {x: Camera.position.x, y: (Camera.position.y - 0.5), z: Camera.position.z};
var myPos6 = {x: Camera.position.x, y: (Camera.position.y - 0.4), z: Camera.position.z};
var forwardDirection = Quat.getFront(Camera.orientation);
forwardDirection.y = 0.0;
var flatForward = Vec3.normalize(forwardDirection);
//var up = Vec3.multiply(0.1,Quat.getUp(Camera.orientation));
//var above = Vec3.sum(up,Camera.position);
var offsetUp = Vec3.sum(Vec3.multiply(0.1, Quat.getUp(Camera.orientation)),myPos);
var forwardEnd = Vec3.sum(myPos,Vec3.multiply(5.0,Quat.getFront(Camera.orientation)));
/*
DebugDraw.drawRay(offsetUp, forwardEnd,{ red:1, blue:0, green:0, alpha:1 });
DebugDraw.drawRay(myPos1, Vec3.sum(myPos1,Vec3.multiply(5.0,flatForward)),{ red:0, blue:1, green:0, alpha:1 });
DebugDraw.drawRay(myPos2, Vec3.sum(myPos2,Vec3.multiply(5.0,flatForward)),{ red:0, blue:1, green:0, alpha:1 });
DebugDraw.drawRay(myPos3, Vec3.sum(myPos3,Vec3.multiply(5.0,flatForward)),{ red:0, blue:1, green:0, alpha:1 });
DebugDraw.drawRay(myPos4, Vec3.sum(myPos4,Vec3.multiply(5.0,flatForward)),{ red:0, blue:1, green:0, alpha:1 });
DebugDraw.drawRay(myPos5, Vec3.sum(myPos5,Vec3.multiply(5.0,flatForward)),{ red:0, blue:1, green:0, alpha:1 });
DebugDraw.drawRay(myPos6, Vec3.sum(myPos6,Vec3.multiply(5.0,flatForward)),{ red:0, blue:1, green:0, alpha:1 });
DebugDraw.addMarker("head", Camera.orientation, forwardEnd,{ red:1, blue:0, green:0, alpha:1 });
DebugDraw.addMarker("pos1", Camera.orientation, Vec3.sum(myPos1,Vec3.multiply(5.0,flatForward)),{ red:1, blue:0, green:0, alpha:1 });
DebugDraw.addMarker("pos2", Camera.orientation, Vec3.sum(myPos2,Vec3.multiply(5.0,flatForward)),{ red:1, blue:0, green:0, alpha:1 });
DebugDraw.addMarker("pos3", Camera.orientation, Vec3.sum(myPos3,Vec3.multiply(5.0,flatForward)),{ red:1, blue:0, green:0, alpha:1 });
DebugDraw.addMarker("pos4", Camera.orientation, Vec3.sum(myPos4,Vec3.multiply(5.0,flatForward)),{ red:1, blue:0, green:0, alpha:1 });
DebugDraw.addMarker("pos5", Camera.orientation, Vec3.sum(myPos5,Vec3.multiply(5.0,flatForward)),{ red:1, blue:0, green:0, alpha:1 });
DebugDraw.addMarker("pos6", Camera.orientation, Vec3.sum(myPos6,Vec3.multiply(5.0,flatForward)),{ red:1, blue:0, green:0, alpha:1 });
*/
}
/********************************
// END Global Detection Start/Stop/Update functions
********************************/
/********************************
// START Global Capture Start/Stop/Update functions
********************************/
var dataCaptureStartTimeMS;
var DATA_CAPTURE_TIMEOUT_MS = 3000;
function dataCaptureUpdateLoop() {
if (Date.now() - dataCaptureStartTimeMS > DATA_CAPTURE_TIMEOUT_MS) {
stopRecordingSelectedGestures();
return;
}
if (jointDataToRecord[0]) {
leftHandRecorder.captureDataNow();
}
if (jointDataToRecord[1]) {
rightHandRecorder.captureDataNow();
}
if (jointDataToRecord[2]) {
headRecorder.captureDataNow();
}
}
// [0] is "left hand"
// [1] is "right hand"
// [2] is "head"
var jointDataToRecord = [
Settings.getValue('gestures/captureLeftHandData', false),
Settings.getValue('gestures/captureRightHandData', false),
Settings.getValue('gestures/captureHeadData', false)
];
var updateConnected = false;
var leftHandRecorder = new GestureRecorder(LEFT_HAND_JOINT_NAME);
var rightHandRecorder = new GestureRecorder(RIGHT_HAND_JOINT_NAME);
var headRecorder = new GestureRecorder(HEAD_JOINT_NAME);
function startRecordingSelectedGestures() {
if (!(jointDataToRecord[0] || jointDataToRecord[1] || jointDataToRecord[2])) {
return;
}
maybePrint("ZRF startRecordingSelectedGestures()", DEBUG_UNIMPORTANT);
isRecordingSelectedGestures = true;
sendToQml({ method: 'updateIsRecordingSelectedGestures', isRecordingSelectedGestures: isRecordingSelectedGestures });
Audio.playSound(SOUND_GESTURE_RECORDING_START, {
position: MyAvatar.position,
localOnly: true,
volume: 0.8
});
if (updateConnected) {
Script.update.disconnect(dataCaptureUpdateLoop);
updateConnected = false;
}
if (jointDataToRecord[0]) {
leftHandRecorder.initializeDataForRecording();
}
if (jointDataToRecord[1]) {
rightHandRecorder.initializeDataForRecording();
}
if (jointDataToRecord[2]) {
headRecorder.initializeDataForRecording();
}
dataCaptureStartTimeMS = Date.now();
Script.update.connect(dataCaptureUpdateLoop);
updateConnected = true;
updateGestureDetectionSystem();
}
function stopRecordingSelectedGestures() {
maybePrint("ZRF stopRecordingSelectedGestures()", DEBUG_UNIMPORTANT);
isRecordingSelectedGestures = false;
if (updateConnected) {
Script.update.disconnect(dataCaptureUpdateLoop);
updateConnected = false;
}
sendToQml({ method: 'updateIsRecordingSelectedGestures', isRecordingSelectedGestures: isRecordingSelectedGestures });
Audio.playSound(SOUND_GESTURE_RECORDING_STOP, {
position: MyAvatar.position,
localOnly: true,
volume: 0.8
});
var timestamp;
var index = -1;
if (jointDataToRecord[0]) {
var dataToSave = {
timestamp: false,
data: []
};
leftHandRecorder.stopRecording();
timestamp = leftHandRecorder.recordingStartTimeMS;
dataToSave.timestamp = timestamp;
dataToSave.data = leftHandRecorder.currentPoseFrameData;
leftHandRecorder.previouslyRecordedFrameData.push(dataToSave);
leftHandRecorder.currentPoseFrameData = [];
index = leftHandRecorder.previouslyRecordedFrameData.length - 1;
} else {
leftHandRecorder.previouslyRecordedFrameData.push("NODATA");
}
if (jointDataToRecord[1]) {
var dataToSave2 = {
timestamp: false,
data: []
};
rightHandRecorder.stopRecording();
timestamp = rightHandRecorder.recordingStartTimeMS;
dataToSave2.timestamp = timestamp;
dataToSave2.data = rightHandRecorder.currentPoseFrameData;
rightHandRecorder.previouslyRecordedFrameData.push(dataToSave2);
rightHandRecorder.currentPoseFrameData = [];
index = rightHandRecorder.previouslyRecordedFrameData.length - 1;
} else {
rightHandRecorder.previouslyRecordedFrameData.push("NODATA");
}
if (jointDataToRecord[2]) {
var dataToSave3 = {
timestamp: false,
data: []
};
headRecorder.stopRecording();
timestamp = headRecorder.recordingStartTimeMS;
dataToSave3.timestamp = timestamp;
dataToSave3.data = headRecorder.currentPoseFrameData;
headRecorder.previouslyRecordedFrameData.push(dataToSave3);
headRecorder.currentPoseFrameData = [];
index = headRecorder.previouslyRecordedFrameData.length - 1;
} else {
headRecorder.previouslyRecordedFrameData.push("NODATA");
}
timestamp = new Date(timestamp);
sendToQml({
method: 'appendRecordedGesture',
index: index,
recordedTime: timestamp.getHours() + ":" + timestamp.getMinutes() + ":" + timestamp.getSeconds(),
recordedJoints: [jointDataToRecord[0], jointDataToRecord[1], jointDataToRecord[2]]
});
updateGestureDetectionSystem();
}
var isRecordingSelectedGestures = false;
function toggleRecordingGesture() {
if (!isRecordingSelectedGestures) {
startRecordingSelectedGestures();
} else {
stopRecordingSelectedGestures();
}
}
/********************************
// END Global Capture Start/Stop/Update functions
********************************/
/********************************
// START Controller Mapping
********************************/
var gesturesControllerMapping = false;
var gesturesControllerMappingName = 'Hifi-Gestures-Mapping';
function maybeRegisterButtonMappings() {
// Don't re-register
if (gesturesControllerMapping) {
return;
}
gesturesControllerMapping = Controller.newMapping(gesturesControllerMappingName);
if (controllerType === "OculusTouch") {
gesturesControllerMapping.from(Controller.Standard.RS).to(function (value) {
if (value === 1.0) {
toggleRecordingGesture();
}
return;
});
} else if (controllerType === "Vive") {
gesturesControllerMapping.from(Controller.Standard.RightPrimaryThumb).to(function (value) {
if (value === 1.0) {
toggleRecordingGesture();
}
return;
});
}
gesturesControllerMapping.enable();
}
function disableButtonMappings() {
if (gesturesControllerMapping) {
gesturesControllerMapping.disable();
gesturesControllerMapping = false;
}
}
/********************************
// END Controller Mapping
********************************/
/********************************
// START App-Related Functions
********************************/
// Function Name: sendToQml()
//
// Description:
// -Use this function to send a message to the app's QML (i.e. to change appearances). The "message" argument is what is sent to
// the app's QML in the format "{method, params}", like json-rpc. See also fromQml().
function sendToQml(message) {
ui.sendMessage(message);
}
// Function Name: fromQml()
//
// Description:
// -Called when a message is received from the app QML. The "message" argument is what is sent from the app QML
// in the format "{method, params}", like json-rpc. See also sendToQml().
function fromQml(message) {
switch (message.method) {
case 'enableRecordingSwitchChanged':
if (message.status) {
maybeRegisterButtonMappings();
} else {
disableButtonMappings();
}
Settings.setValue('gestures/enableControllerMapping', message.status);
break;
case 'updateJointSelections':
jointDataToRecord = message.selections;
Settings.setValue('gestures/captureLeftHandData', jointDataToRecord[0]);
Settings.setValue('gestures/captureRightHandData', jointDataToRecord[1]);
Settings.setValue('gestures/captureHeadData', jointDataToRecord[2]);
break;
case 'toggleManualDataCapture':
toggleRecordingGesture();
break;
case 'modifyListeningForGestureIndex':
// console.log("the gesture index is set to " + message.listeningForGestureIndex);
gestureToDetect_index = message.listeningForGestureIndex;
updateGestureDetectionSystem();
break;
case 'clearRecordedGestures':
gestureToDetect_index = -1;
leftHandRecorder.previouslyRecordedFrameData = [];
rightHandRecorder.previouslyRecordedFrameData = [];
headRecorder.previouslyRecordedFrameData = [];
break;
case 'updateSensitivity':
gestureDetectionSensitivityMultiplier = message.sensitivity;
Settings.setValue('gestures/sensitivity', gestureDetectionSensitivityMultiplier);
break;
case 'updateMaxIgnoreDistance':
avatarIgnoreMaxDistance = message.distance;
Settings.setValue('gestures/maxIgnoreDistance', avatarIgnoreMaxDistance);
maybePrint("ZRF: Updating max avatar ignore distance to '" + avatarIgnoreMaxDistance, DEBUG_IMPORTANT);
break;
case 'copyDataToClipboard':
var dataToCopy = [];
var leftHandData = "NODATA";
var rightHandData = "NODATA";
var headData = "NODATA";
if (leftHandRecorder.previouslyRecordedFrameData[message.index] !== "NODATA") {
leftHandData = leftHandRecorder.previouslyRecordedFrameData[message.index];
}
if (rightHandRecorder.previouslyRecordedFrameData[message.index] !== "NODATA") {
rightHandData = rightHandRecorder.previouslyRecordedFrameData[message.index];
}
if (headRecorder.previouslyRecordedFrameData[message.index] !== "NODATA") {
headData = headRecorder.previouslyRecordedFrameData[message.index];
}
dataToCopy.push(leftHandData);
dataToCopy.push(rightHandData);
dataToCopy.push(headData);
Window.copyToClipboard(JSON.stringify(dataToCopy));
break;
case 'importGestureData':
var data = JSON.parse(message.data);
if (data.length !== 3) {
maybePrint('Unrecognized import data format, bailing!', DEBUG_URGENT);
return;
}
leftHandRecorder.previouslyRecordedFrameData.push(data[0]);
rightHandRecorder.previouslyRecordedFrameData.push(data[1]);
headRecorder.previouslyRecordedFrameData.push(data[2]);
appendGestureToQMLWithIndex(rightHandRecorder.previouslyRecordedFrameData.length - 1);
break;
default:
maybePrint('Unrecognized message from Gestures.qml: ' + JSON.stringify(message), DEBUG_URGENT);
}
}
function appendGestureToQMLWithIndex(index) {
var timestamp;
var recordedJoints = [false, false, false];
if (leftHandRecorder.previouslyRecordedFrameData[index] !== "NODATA") {
recordedJoints[0] = true;
timestamp = leftHandRecorder.previouslyRecordedFrameData[index].timestamp;
}
if (rightHandRecorder.previouslyRecordedFrameData[index] !== "NODATA") {
recordedJoints[1] = true;
timestamp = rightHandRecorder.previouslyRecordedFrameData[index].timestamp;
}
if (headRecorder.previouslyRecordedFrameData[index] !== "NODATA") {
recordedJoints[2] = true;
timestamp = headRecorder.previouslyRecordedFrameData[index].timestamp;
}
timestamp = new Date(timestamp);
sendToQml({
method: 'appendRecordedGesture',
index: index,
recordedTime: timestamp.getHours() + ":" + timestamp.getMinutes() + ":" + timestamp.getSeconds(),
recordedJoints: recordedJoints
});
}
function appendAllGesturesToQML() {
for (var i = 0; i < leftHandRecorder.previouslyRecordedFrameData.length; i++) {
appendGestureToQMLWithIndex(i);
}
}
// Function Name: appUiOpened()
//
// Description:
// - Called when the app's UI is opened
//
var APP_INITIALIZE_UI_DELAY = 500; // MS
function appUiOpened() {
// In the case of a remote QML app, it takes a bit of time
// for the event bridge to actually connect, so we have to wait...
Script.setTimeout(function () {
sendToQml({
method: 'initializeUI',
masterSwitchOn: !!gesturesControllerMapping,
jointDataToRecord: jointDataToRecord,
currentlyDetectingGesture: gestureToDetect_index,
gestureDetectionSensitivityMultiplier: gestureDetectionSensitivityMultiplier,
maxIgnoreDistance: avatarIgnoreMaxDistance
});
appendAllGesturesToQML();
}, APP_INITIALIZE_UI_DELAY);
}
// Function Name: appUiClosed()
//
// Description:
// - Called when the app's UI is closed
//
function appUiClosed() {
}
// Function Name: startup()
//
// Description:
// -startup() will be called when the script is loaded.
//
var ui;
var controllerType = "Other";
var avatarIgnoreMaxDistance = 5.0;
function startup() {
ui = new AppUi({
buttonName: "GESTURES",
home: Script.resolvePath('https://hifi-content.s3.amazonaws.com/zfox/gestureApp/Gestures.qml'),
onOpened: appUiOpened,
onClosed: appUiClosed,
onMessage: fromQml,
sortOrder: 15,
normalButton: Script.resourcesPath() + "icons/tablet-icons/avatar-record-i.svg",
activeButton: Script.resourcesPath() + "icons/tablet-icons/avatar-record-a.svg"
});
// Controller type detection
var VRDevices = Controller.getDeviceNames().toString();
if (VRDevices) {
if (VRDevices.indexOf("Vive") !== -1) {
controllerType = "Vive";
} else if (VRDevices.indexOf("OculusTouch") !== -1) {
controllerType = "OculusTouch";
}
}
if (Settings.getValue('gestures/enableControllerMapping', false)) {
maybeRegisterButtonMappings();
}
avatarIgnoreMaxDistance = Settings.getValue('gestures/maxIgnoreDistance', 5.0);
var borderModelProperties = {
position: getPositionForIcon(),
lifetime: 100,
modelURL: "file:///c:/angus/javascript_bak/gesture/outline.fbx",
name: " border",
type : "Model"
};
var loadingBarModelProperties = {
position: getPositionForIcon(),
lifetime: 100,
dimensions: {
x: 0.001,
y: 0.1,
z: 0.1
},
modelURL: "file:///c:/angus/javascript_bak/gesture/loadingBar.fbx",
name: " border",
type : "Model"
};
//loadingBarEntityID = Entities.addEntity(loadingBarModelProperties);
//borderEntityID = Entities.addEntity(borderModelProperties);
//Entities.editEntity(borderEntityID, {dimensions: { x: 0.001, y: 0.1, z: 0.5 }});
}
// Function Name: shutdown()
//
// Description:
// - Called when the script ends (i.e. is stopped).
//
function shutdown() {
appUiClosed();
//Entities.deleteEntity(borderEntityID);
//Entities.deleteEntity(loadingBarEntityID);
Overlays.deleteOverlay(animShush.uuid);
Overlays.deleteOverlay(animShush.border);
Overlays.deleteOverlay(animShush.loadBar);
Overlays.deleteOverlay(animShush.textUUID);
/*
DebugDraw.removeMarker("head");
DebugDraw.removeMarker("pos1");
DebugDraw.removeMarker("pos2");
DebugDraw.removeMarker("pos3");
DebugDraw.removeMarker("pos4");
DebugDraw.removeMarker("pos5");
DebugDraw.removeMarker("pos6");
*/
}
var SOUND_GESTURE_RECORDING_START = SoundCache.getSound(Script.resolvePath("https://hifi-content.s3.amazonaws.com/zfox/gestureApp/startRecording.wav"));
var SOUND_GESTURE_RECORDING_STOP = SoundCache.getSound(Script.resolvePath("https://hifi-content.s3.amazonaws.com/zfox/gestureApp/stopRecording.wav"));
var SOUND_GESTURE_DETECTED = SoundCache.getSound(Script.resolvePath("https://hifi-content.s3.amazonaws.com/zfox/gestureApp/gestureDetected.wav"));
startup();
Script.scriptEnding.connect(shutdown);
/********************************
// END App-Related Functions
********************************/
}()); // END LOCAL_SCOPE