"use strict"; /*jslint vars:true, plusplus:true, forin:true*/ /*global Tablet, Script, Users, console */ /* eslint indent: ["error", 4, { "outerIIFEBody": 0 }] */ // // gestures.js // // Created by Zach Fox on 2018-07-24 // Copyright 2018 High Fidelity, Inc // // Distributed under the Apache License, Version 2.0 // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // (function () { // BEGIN LOCAL_SCOPE var AppUi = Script.require('appUi'); /******************************** // START Debug Functions ********************************/ var DEBUG_UNIMPORTANT = 0; var DEBUG_IMPORTANT = 1; var DEBUG_URGENT = 2; var DEBUG_ONLY_PRINT_URGENT = 0; var DEBUG_PRINT_URGENT_AND_IMPORTANT = 1; var DEBUG_PRINT_EVERYTHING = 2; var debugLevel = DEBUG_ONLY_PRINT_URGENT; var avatarIgnoreID = null; var borderEntityID = null; var loadingBarEntityID = null; var isBlockFocus = true; var blockFurtherDetection = false; function maybePrint(string, importance) { if (importance >= (DEBUG_URGENT - debugLevel)) { console.log(string); } } /******************************** // END Debug Functions ********************************/ /******************************** //overlay code *******************************/ var ROOT = "http://mpassets.highfidelity.com/62f401b8-cc95-4e18-8723-12f076e6e5fd-v1/"; var startFrame = 1; var inFrontOfMyAvatar = getPositionForIcon(); inFrontOfMyAvatar.x = inFrontOfMyAvatar.x - 0.51; var animation = { setupAnimation: function (url, width, height, maxFrames, fps, text) { maxFrames = maxFrames || 1; var anim = { timer: null, url: url, frames: {}, running: false, fps: 1000 / fps, currentFrame: startFrame, maxFrames: maxFrames, width: width, height: height, emissive: true, text: text, blockedYet: false }; for (var i = startFrame; i <= maxFrames; i++) { anim.frames[i] = TextureCache.prefetch(url.slice(0, -4) + i + ".png"); } anim.border = Overlays.addOverlay("image3d", { position: getPositionForIcon(), drawInFront: true, isFacingAvatar: true, url: "http://hifi-content.s3.amazonaws.com/angus/gesture/outline.png", alpha: 0.0, ignorePickIntersection: true, parentID: MyAvatar.sessionUUID }); anim.loadBar = Overlays.addOverlay("image3d", { parentID: anim.border, localPosition: {x: -0.48, y: 0, z: 0}, // getPositionForIcon(), dimensions: {x: 0.0, y: 0.5}, drawInFront: false, isFacingAvatar: true, url: "http://hifi-content.s3.amazonaws.com/angus/gesture/loadingBar.png", alpha: 0.0, ignorePickIntersection: true }); anim.uuid = Overlays.addOverlay("image3d", { position: getPositionForIcon(), drawInFront: true, dimensions: {x: 0.5, y: 0.5}, isFacingAvatar: true, url: anim.frames[startFrame].url, alpha: 0, ignorePickIntersection: true, parentID: MyAvatar.sessionUUID }); anim.textUUID = Overlays.addOverlay("text3d", { localPosition: {x: -(anim.text.length * 0.00875), y: 0.0, z: 0}, parentID: anim.border, drawInFront: true, dimensions: {x: 0.0, y: 0.0}, leftMargin: -0.04, topMargin: -0.01, isFacingAvatar: true, text: anim.text, lineHeight: 0.07, textAlpha: 0, alpha: 0, ignorePickIntersection: true, fontsize: 36 }); anim.end = function (stopped) { Script.clearInterval(anim.timer); anim.running = false; anim.currentFrame = startFrame; Overlays.editOverlay(anim.uuid, {alpha: 0}); Overlays.editOverlay(anim.textUUID, {textAlpha: 0}); Overlays.editOverlay(anim.border, {alpha: 0}); Overlays.editOverlay(anim.loadBar, {alpha: 0}); //if (!stopped) { // anim.callback(anim.callbackData); //} anim.blockedYet = false; }; anim.stop = function () { anim.end(true); }; anim.nextFrame = function () { anim.currentFrame++; if (anim.currentFrame >= anim.maxFrames) { if (anim.running) { anim.end(false); } } if (anim.currentFrame < (anim.maxFrames - 20)) { Overlays.editOverlay(anim.loadBar, { localPosition: { x: - (0.47 - (anim.currentFrame/(anim.maxFrames-21))*0.47), y: 0, z: 0 }, dimensions: {x: (anim.currentFrame/(anim.maxFrames-21))*0.97, y: 0.48} }); Overlays.editOverlay(anim.border, { position: getPositionForIcon() }); } else { Overlays.editOverlay(anim.textUUID, { text: "USER BLOCKED" }); if (!anim.blockedYet) { anim.blockedYet = true; anim.callback(anim.callbackData); } } var propertiesToGet = {}; propertiesToGet[anim.uuid] = ["rotation","parentID"]; var props = Overlays.getOverlaysProperties(propertiesToGet); }; anim.start = function (position, _callback, _callbackData) { if (!anim.running) { anim.running = true; anim.callback = _callback; anim.callbackData = _callbackData; Overlays.editOverlay(anim.uuid, { url: anim.frames[startFrame].url, position: position, alpha: 0 }); Overlays.editOverlay(anim.textUUID, { textAlpha: 1, text: anim.text, }); Overlays.editOverlay(anim.border, { position: position, alpha: 1 }); Overlays.editOverlay(anim.loadBar, { localPosition: {x: -0.47, y: 0, z: 0}, alpha: 1 }); if (anim.timer) { Script.clearInterval(anim.timer); } anim.timer = Script.setInterval(anim.nextFrame, anim.fps); } }; return anim; } }; var animShush = animation.setupAnimation(ROOT + "loading/loading.png", 200, 200, 60, 30, "Hold to Block"); function getPositionForIcon() { var camPos = Camera.position; var camRot = Camera.orientation; return Vec3.sum(camPos, Vec3.multiplyQbyV(camRot, {x: 0, y: 0, z: -1})); } function shushPerson(hand) { if ((avatarIgnoreID !== null) && !isBlockFocus) { Users.ignore(avatarIgnoreID); avatarIgnoreID = null; isBlockFocus = true; } blockFurtherDetection = false; print("sush timer expired"); } /******************************** // START GestureRecorder ********************************/ var LEFT_HAND_JOINT_NAME = "leftHand"; var RIGHT_HAND_JOINT_NAME = "rightHand"; var HEAD_JOINT_NAME = "head"; function GestureRecorder(jointName) { this.jointName = jointName; this.currentPoseFrameData = []; this.previouslyRecordedFrameData = []; this.recordingStartTimeMS; this.recordingLengthSEC; this.isRecordingGesture = false; } GestureRecorder.prototype.initializeDataForRecording = function () { this.currentPoseFrameData = []; this.recordingStartTimeMS = Date.now(); maybePrint("ZRF: Starting gesture recording for joint '" + this.jointName + "'.", DEBUG_IMPORTANT); }; var Y_AXIS = { x: 0, y: 1, z: 0 }; GestureRecorder.prototype.sampleFrame = function (timeSinceStartSEC, previousSample) { var pose; var headPose = Controller.getPoseValue(Controller.Standard.Head); if (this.jointName === LEFT_HAND_JOINT_NAME) { pose = Controller.getPoseValue(Controller.Standard.LeftHand); } else if (this.jointName === RIGHT_HAND_JOINT_NAME) { pose = Controller.getPoseValue(Controller.Standard.RightHand); pose.rotation = Quat.multiply(Quat.inverse(headPose.rotation), pose.rotation); var handOffset = Vec3.subtract(pose.translation, headPose.translation); pose.translation = Vec3.multiplyQbyV(Quat.inverse(headPose.rotation), handOffset); var headDotHand = Vec3.dot({x: 0.0,y: 0.0,z: 1.0}, Vec3.normalize(pose.translation)); if (headDotHand < 0.866) { isBlockFocus = true; if (animShush.running) { animShush.stop(); } blockFurtherDetection = false; } // print("isBlockFocus " + isBlockFocus); } else if (this.jointName === HEAD_JOINT_NAME) { pose = Controller.getPoseValue(Controller.Standard.Head); // pose.rotation = Quat.fromVec3Degrees({ x: 0.0, y: 0.0, z: 0.0 }); // pose.translation = { x: 0.0, y: 0.0, z: 0.0 }; } // make things relative to the head // pose.rotation = Quat.multiply(Quat.inverse(headPose.rotation), pose.rotation); // pose.translation = Vec3.subtract(pose.translation, headPose.translation); var frameData = { timeSinceStartSEC: timeSinceStartSEC, x: pose.translation.x, y: pose.translation.y, z: pose.translation.z, rotation: pose.rotation }; return frameData; }; var MS_PER_SEC = 1000; GestureRecorder.prototype.captureDataNow = function () { var now = Date.now(); var timeSinceStartSEC = (now - this.recordingStartTimeMS) / MS_PER_SEC; this.currentPoseFrameData.push(this.sampleFrame(timeSinceStartSEC)); }; GestureRecorder.prototype.stopRecording = function () { this.recordingLengthSEC = (Date.now() - this.recordingStartTimeMS) / MS_PER_SEC; calculateDerivatives(this.currentPoseFrameData); maybePrint("ZRF: Finished gesture recording for joint '" + this.jointName + "'.", DEBUG_IMPORTANT); JSON.stringify(this.currentPoseFrameData, null, 4).split("\n").forEach(function (str) { maybePrint(str, DEBUG_UNIMPORTANT); }); }; GestureRecorder.prototype.gestureDetectCheck = function () { maybePrint("gestureDetect() currentPoseFrameData.length = " + this.currentPoseFrameData.length + ", previouslyRecordedFrameData.data.length = " + this.previouslyRecordedFrameData[gestureToDetect_index].data.length + ", recordingLength = " + this.recordingLengthSEC + " (sec)", DEBUG_UNIMPORTANT); // not enough frames to test if (this.currentPoseFrameData.length < this.previouslyRecordedFrameData[gestureToDetect_index].data.length / 2) { return false; } var i = 0, j = 0; var framesTested = 0; var framesPassed = 0; while (i < this.previouslyRecordedFrameData[gestureToDetect_index].data.length && j < this.currentPoseFrameData.length) { var it = this.previouslyRecordedFrameData[gestureToDetect_index].data[i].timeSinceStartSEC; var jt = this.currentPoseFrameData[j].timeSinceStartSEC - this.currentPoseFrameData[0].timeSinceStartSEC; if (jt < it) { var iPrev = i > 0 ? (i - 1) : 0; var a = this.previouslyRecordedFrameData[gestureToDetect_index].data[iPrev]; var b = this.previouslyRecordedFrameData[gestureToDetect_index].data[i]; var alpha = i > 0 ? (jt - a.timeSinceStartSEC) / (b.timeSinceStartSEC - a.timeSinceStartSEC) : 0; var frame = lerpFrame(a, b, alpha); framesTested++; if (testFrames(this.currentPoseFrameData[j], frame)) { framesPassed++; } j++; } else { i++; } } // console.log("frames tested and passed " + framesTested + " " + framesPassed ); if (framesPassed / framesTested > 0.4) { maybePrint(~~((framesPassed / framesTested) * 100) + "% match", DEBUG_UNIMPORTANT); } return framesPassed / framesTested > 0.75; }; GestureRecorder.prototype.gestureDetect = function () { var t = Date.now() / MS_PER_SEC; this.currentPoseFrameData.push(this.sampleFrame(t)); calculateDerivatives(this.currentPoseFrameData); var dataLength = this.previouslyRecordedFrameData[gestureToDetect_index].data.length; var i; for (i = 0; i < this.currentPoseFrameData.length; i++) { if (i > 0 && this.currentPoseFrameData[i].timeSinceStartSEC > t - this.previouslyRecordedFrameData[gestureToDetect_index].data[dataLength-1].timeSinceStartSEC) { this.currentPoseFrameData = this.currentPoseFrameData.slice(i - 1); break; } } // console.log("length of recording " + this.previouslyRecordedFrameData[gestureToDetect_index].data[dataLength-1].timeSinceStartSEC + " length of currentData " + this.currentPoseFrameData.length); if (this.currentPoseFrameData.length > 0 && this.gestureDetectCheck()) { maybePrint("ZRF: Gesture with index " + gestureToDetect_index + " detected on joint " + this.jointName + "!", DEBUG_UNIMPORTANT); return true; } return false; }; /******************************** // END GestureRecorder ********************************/ /******************************** // START Shared Math Utility Functions ********************************/ // Function Name: inFrontOf() // // Description: // -Returns the position in front of the given "position" argument, where the forward vector is based off // the "orientation" argument and the amount in front is based off the "distance" argument. function inFrontOf(distance, position, orientation) { return Vec3.sum(position || MyAvatar.position, Vec3.multiply(distance, Quat.getForward(orientation || MyAvatar.orientation))); } function calculateDerivatives(frames) { var i, length = frames.length; var keys = ["x", "y", "z"]; var dKeys = ["dx", "dy", "dz"]; for (i = 0; i < length; i++) { var prevIndex = (i === 0) ? 0 : i - 1; var nextIndex = (i === length - 1) ? i : i + 1; var j = 0, numKeys = keys.length; for (j = 0; j < numKeys; j++) { var d1 = frames[i][keys[j]] - frames[prevIndex][keys[j]]; var d2 = frames[nextIndex][keys[j]] - frames[i][keys[j]]; frames[i][dKeys[j]] = (d1 + d2) / 2; } } } function lerp(a, b, alpha) { return a * (1 - alpha) + b * alpha; } function lerpFrame(a, b, alpha) { var keys = Object.keys(a); var result = {}; keys.forEach(function (key) { result[key] = lerp(a[key], b[key], alpha); }); return result; } var RAD_TO_DEG = 180 / Math.PI; var DEG_TO_RAD = Math.PI / 180; var ROTATION_THRESHOLD = 30.0 * DEG_TO_RAD; // radians var X_THRESHOLD = 0.2; // meters var Y_THRESHOLD = 0.2; // meters var Z_THRESHOLD = 0.2; // meters var DX_THRESHOLD = 0.02; // meters / sec var DY_THRESHOLD = 0.02; // meters / sec var DZ_THRESHOLD = 0.02; // meters / sec function testFrames(a, b) { //console.log("gesture test frames"); if (Math.abs(Quat.dot(a.rotation, b.rotation)) < Math.cos(ROTATION_THRESHOLD * gestureDetectionSensitivityMultiplier)) { return false; } if (Math.abs(a.x - b.x) > X_THRESHOLD * gestureDetectionSensitivityMultiplier) { return false; } if (Math.abs(a.y - b.y) > Y_THRESHOLD * gestureDetectionSensitivityMultiplier) { return false; } if (Math.abs(a.z - b.z) > Z_THRESHOLD * gestureDetectionSensitivityMultiplier) { return false; } if (Math.abs(a.dx - b.dx) > DX_THRESHOLD * gestureDetectionSensitivityMultiplier) { return false; } if (Math.abs(a.dy - b.dy) > DY_THRESHOLD * gestureDetectionSensitivityMultiplier) { return false; } if (Math.abs(a.dz - b.dz) > DZ_THRESHOLD * gestureDetectionSensitivityMultiplier) { return false; } return true; } /******************************** // END Shared Math Utility Functions ********************************/ /******************************** // START Global Detection Start/Stop/Update functions ********************************/ var isDetectingGesture = false; var gestureToDetect_index = -1; var gestureDetectionSensitivityMultiplier = Settings.getValue('gestures/sensitivity', 1.0); function updateGestureDetectionSystem() { if (gestureToDetect_index > -1 && !isRecordingSelectedGestures) { if (isDetectingGesture) { Script.update.disconnect(gestureDetectionUpdateLoop); isDetectingGesture = false; } isDetectingGesture = true; Script.update.connect(gestureDetectionUpdateLoop); } else { if (isDetectingGesture) { Script.update.disconnect(gestureDetectionUpdateLoop); isDetectingGesture = false; leftHandRecorder.currentPoseFrameData = []; rightHandRecorder.currentPoseFrameData = []; headRecorder.currentPoseFrameData = []; } } maybePrint("ZRF: The gesture to detect has index: " + gestureToDetect_index + ". Currently detecting gestures: " + isDetectingGesture, DEBUG_IMPORTANT); } var detectedEntity = false; var deleteDetectedEntityTimeout = false; function handleDetectedEntity() { if (!detectedEntity) { detectedEntity = Entities.addEntity({ "collidesWith": "", "collisionMask": 0, "collisionless": true, "color": { "blue": 20, "green": 200, "red": 20 }, "dimensions": { "blue": 0.05000000074505806, "green": 0.4000000059604645, "red": 0.4000000059604645, "x": 0.4000000059604645, "y": 0.4000000059604645, "z": 0.05000000074505806 }, "ignoreForCollisions": true, "shape": "Cube", "type": "Box", "userData": "{\"grabbableKey\":{\"grabbable\":false}}", "position": inFrontOf(0.8, Camera.position, Camera.orientation), "rotation": Camera.orientation }, true); } if (deleteDetectedEntityTimeout) { Script.clearTimeout(deleteDetectedEntityTimeout); } deleteDetectedEntityTimeout = Script.setTimeout(function () { if (detectedEntity) { Entities.deleteEntity(detectedEntity); detectedEntity = false; } deleteDetectedEntityTimeout = false; }, 1000); } function handleEntityPickRay() { var pickRay = { origin: Camera.position, direction: Quat.getFront(Camera.orientation), length: 100 }; var entityIntersection = Entities.findRayIntersection(pickRay, true); if (entityIntersection.intersects) { var intersectEntityID = entityIntersection.entityID; Entities.editEntity(intersectEntityID, { color: { red: Math.random() * 255, green: Math.random() * 255, blue: Math.random() * 255 } }); } } function handleAvatarPickRay() { var myPos = Camera.position; var myPos1 = {x: Camera.position.x, y: (Camera.position.y - 1.3), z: Camera.position.z}; var myPos2 = {x: Camera.position.x, y: (Camera.position.y - 1.1), z: Camera.position.z}; var myPos3 = {x: Camera.position.x, y: (Camera.position.y - 0.9), z: Camera.position.z}; var myPos4 = {x: Camera.position.x, y: (Camera.position.y - 0.7), z: Camera.position.z}; var myPos5 = {x: Camera.position.x, y: (Camera.position.y - 0.5), z: Camera.position.z}; var myPos6 = {x: Camera.position.x, y: (Camera.position.y - 0.4), z: Camera.position.z}; var forwardDirection = Quat.getFront(Camera.orientation); forwardDirection.y = 0.0; var flatForward = Vec3.normalize(forwardDirection); var offsetUp = Vec3.sum(Vec3.multiply(0.1, Quat.getUp(Camera.orientation)),myPos); var forwardEnd = Vec3.multiply(5.0,Quat.getFront(Camera.orientation)); var pickRay = { origin: Camera.position, direction: Quat.getFront(Camera.orientation), length: avatarIgnoreMaxDistance }; var pickRay1 = { origin: myPos1, direction: flatForward, length: avatarIgnoreMaxDistance }; var pickRay2 = { origin: myPos2, direction: flatForward, length: avatarIgnoreMaxDistance }; var pickRay3 = { origin: myPos3, direction: flatForward, length: avatarIgnoreMaxDistance }; var pickRay4 = { origin: myPos4, direction: flatForward, length: avatarIgnoreMaxDistance }; var pickRay5 = { origin: myPos5, direction: flatForward, length: avatarIgnoreMaxDistance }; var pickRay6 = { origin: myPos2, direction: flatForward, length: avatarIgnoreMaxDistance }; var avatarIntersection = AvatarList.findRayIntersection(pickRay, [], [MyAvatar.sessionUUID]); var avatarIntersection1 = AvatarList.findRayIntersection(pickRay1, [], [MyAvatar.sessionUUID]); var avatarIntersection2 = AvatarList.findRayIntersection(pickRay2, [], [MyAvatar.sessionUUID]); var avatarIntersection3 = AvatarList.findRayIntersection(pickRay3, [], [MyAvatar.sessionUUID]); var avatarIntersection4 = AvatarList.findRayIntersection(pickRay4, [], [MyAvatar.sessionUUID]); var avatarIntersection5 = AvatarList.findRayIntersection(pickRay5, [], [MyAvatar.sessionUUID]); var avatarIntersection6 = AvatarList.findRayIntersection(pickRay6, [], [MyAvatar.sessionUUID]); var ret = false; // console.log("camera position" + JSON.stringify(Camera.position) ); if (avatarIntersection.intersects) { avatarIgnoreID = avatarIntersection.avatarID; ret = true; console.log("we have a hit"); } else if (avatarIntersection1.intersects) { avatarIgnoreID = avatarIntersection1.avatarID; ret = true; console.log("we have a hit--1"); } else if (avatarIntersection2.intersects) { avatarIgnoreID = avatarIntersection2.avatarID; ret = true; console.log("we have a hit----2"); } else if (avatarIntersection3.intersects) { avatarIgnoreID = avatarIntersection3.avatarID; ret = true; console.log("we have a hit------3"); } else if (avatarIntersection4.intersects) { avatarIgnoreID = avatarIntersection4.avatarID; ret = true; console.log("we have a hit--------4"); } else if (avatarIntersection5.intersects) { avatarIgnoreID = avatarIntersection5.avatarID; ret = true; console.log("we have a hit----------5"); } else if (avatarIntersection6.intersects) { avatarIgnoreID = avatarIntersection6.avatarID; ret = true; console.log("we have a hit------------6"); } return ret; } function performGestureDetectedAction() { //maybePrint("ZRF: GESTURE WITH INDEX " + gestureToDetect_index + " DETECTED!", DEBUG_URGENT); sendToQml({ method: 'gestureDetected' }); //handleDetectedEntity(); handleEntityPickRay(); var hitAvatar = handleAvatarPickRay(); // if (hitAvatar) { console.log("did we pick an avatar? " + hitAvatar); //Audio.playSound(SOUND_GESTURE_DETECTED, { // position: MyAvatar.position, // localOnly: true, // volume: 0.3 //}); isBlockFocus = false; animShush.start(getPositionForIcon(), shushPerson, 0); } else { blockFurtherDetection = false; } // print("avatar is in my view " + hitAvatar); } function gestureDetectionUpdateLoop() { var detected = [false, false, false]; var mustBeDetected = [ leftHandRecorder.previouslyRecordedFrameData[gestureToDetect_index] !== "NODATA", rightHandRecorder.previouslyRecordedFrameData[gestureToDetect_index] !== "NODATA", headRecorder.previouslyRecordedFrameData[gestureToDetect_index] !== "NODATA" ]; if (mustBeDetected[0]) { detected[0] = leftHandRecorder.gestureDetect(); } if (mustBeDetected[1]) { detected[1] = rightHandRecorder.gestureDetect(); } if (mustBeDetected[2]) { detected[2] = headRecorder.gestureDetect(); } var allNecessaryGesturesDetected = false; for (var i = 0; i < 3; i++) { if (detected[i] !== mustBeDetected[i]) { allNecessaryGesturesDetected = false; break; } else { allNecessaryGesturesDetected = true; } } //var GREEN = { r: 0, g: 1, b: 0, a: 1 }; //var lookingRay = Vec3.multiply(5.0, Quat.getForward(Camera.orientation)); //DebugDraw.drawRay(Camera.position, Vec3.sum(Camera.position,lookingRay), GREEN); // console.log(" block further detection " + allNecessaryGesturesDetected + " " + blockFurtherDetection); if (allNecessaryGesturesDetected && !blockFurtherDetection) { blockFurtherDetection = true; performGestureDetectedAction(); leftHandRecorder.currentPoseFrameData = []; rightHandRecorder.currentPoseFrameData = []; headRecorder.currentPoseFrameData = []; } var myPos = Camera.position; var myPos1 = {x: Camera.position.x, y: (Camera.position.y - 1.3), z: Camera.position.z}; var myPos2 = {x: Camera.position.x, y: (Camera.position.y - 1.1), z: Camera.position.z}; var myPos3 = {x: Camera.position.x, y: (Camera.position.y - 0.9), z: Camera.position.z}; var myPos4 = {x: Camera.position.x, y: (Camera.position.y - 0.7), z: Camera.position.z}; var myPos5 = {x: Camera.position.x, y: (Camera.position.y - 0.5), z: Camera.position.z}; var myPos6 = {x: Camera.position.x, y: (Camera.position.y - 0.4), z: Camera.position.z}; var forwardDirection = Quat.getFront(Camera.orientation); forwardDirection.y = 0.0; var flatForward = Vec3.normalize(forwardDirection); //var up = Vec3.multiply(0.1,Quat.getUp(Camera.orientation)); //var above = Vec3.sum(up,Camera.position); var offsetUp = Vec3.sum(Vec3.multiply(0.1, Quat.getUp(Camera.orientation)),myPos); var forwardEnd = Vec3.sum(myPos,Vec3.multiply(avatarIgnoreMaxDistance,Quat.getFront(Camera.orientation))); DebugDraw.drawRay(offsetUp, forwardEnd,{ red:1, blue:0, green:0, alpha:1 }); DebugDraw.drawRay(myPos1, Vec3.sum(myPos1,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:0, blue:1, green:0, alpha:1 }); DebugDraw.drawRay(myPos2, Vec3.sum(myPos2,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:0, blue:1, green:0, alpha:1 }); DebugDraw.drawRay(myPos3, Vec3.sum(myPos3,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:0, blue:1, green:0, alpha:1 }); DebugDraw.drawRay(myPos4, Vec3.sum(myPos4,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:0, blue:1, green:0, alpha:1 }); DebugDraw.drawRay(myPos5, Vec3.sum(myPos5,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:0, blue:1, green:0, alpha:1 }); DebugDraw.drawRay(myPos6, Vec3.sum(myPos6,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:0, blue:1, green:0, alpha:1 }); DebugDraw.addMarker("head", Camera.orientation, forwardEnd,{ red:1, blue:0, green:0, alpha:1 }); DebugDraw.addMarker("pos1", Camera.orientation, Vec3.sum(myPos1,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:1, blue:0, green:0, alpha:1 }); DebugDraw.addMarker("pos2", Camera.orientation, Vec3.sum(myPos2,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:1, blue:0, green:0, alpha:1 }); DebugDraw.addMarker("pos3", Camera.orientation, Vec3.sum(myPos3,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:1, blue:0, green:0, alpha:1 }); DebugDraw.addMarker("pos4", Camera.orientation, Vec3.sum(myPos4,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:1, blue:0, green:0, alpha:1 }); DebugDraw.addMarker("pos5", Camera.orientation, Vec3.sum(myPos5,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:1, blue:0, green:0, alpha:1 }); DebugDraw.addMarker("pos6", Camera.orientation, Vec3.sum(myPos6,Vec3.multiply(avatarIgnoreMaxDistance,flatForward)),{ red:1, blue:0, green:0, alpha:1 }); } /******************************** // END Global Detection Start/Stop/Update functions ********************************/ /******************************** // START Global Capture Start/Stop/Update functions ********************************/ var dataCaptureStartTimeMS; var DATA_CAPTURE_TIMEOUT_MS = 3000; function dataCaptureUpdateLoop() { if (Date.now() - dataCaptureStartTimeMS > DATA_CAPTURE_TIMEOUT_MS) { stopRecordingSelectedGestures(); return; } if (jointDataToRecord[0]) { leftHandRecorder.captureDataNow(); } if (jointDataToRecord[1]) { rightHandRecorder.captureDataNow(); } if (jointDataToRecord[2]) { headRecorder.captureDataNow(); } } // [0] is "left hand" // [1] is "right hand" // [2] is "head" var jointDataToRecord = [ Settings.getValue('gestures/captureLeftHandData', false), Settings.getValue('gestures/captureRightHandData', false), Settings.getValue('gestures/captureHeadData', false) ]; var updateConnected = false; var leftHandRecorder = new GestureRecorder(LEFT_HAND_JOINT_NAME); var rightHandRecorder = new GestureRecorder(RIGHT_HAND_JOINT_NAME); var headRecorder = new GestureRecorder(HEAD_JOINT_NAME); function startRecordingSelectedGestures() { if (!(jointDataToRecord[0] || jointDataToRecord[1] || jointDataToRecord[2])) { return; } maybePrint("ZRF startRecordingSelectedGestures()", DEBUG_UNIMPORTANT); isRecordingSelectedGestures = true; sendToQml({ method: 'updateIsRecordingSelectedGestures', isRecordingSelectedGestures: isRecordingSelectedGestures }); Audio.playSound(SOUND_GESTURE_RECORDING_START, { position: MyAvatar.position, localOnly: true, volume: 0.8 }); if (updateConnected) { Script.update.disconnect(dataCaptureUpdateLoop); updateConnected = false; } if (jointDataToRecord[0]) { leftHandRecorder.initializeDataForRecording(); } if (jointDataToRecord[1]) { rightHandRecorder.initializeDataForRecording(); } if (jointDataToRecord[2]) { headRecorder.initializeDataForRecording(); } dataCaptureStartTimeMS = Date.now(); Script.update.connect(dataCaptureUpdateLoop); updateConnected = true; updateGestureDetectionSystem(); } function stopRecordingSelectedGestures() { maybePrint("ZRF stopRecordingSelectedGestures()", DEBUG_UNIMPORTANT); isRecordingSelectedGestures = false; if (updateConnected) { Script.update.disconnect(dataCaptureUpdateLoop); updateConnected = false; } sendToQml({ method: 'updateIsRecordingSelectedGestures', isRecordingSelectedGestures: isRecordingSelectedGestures }); Audio.playSound(SOUND_GESTURE_RECORDING_STOP, { position: MyAvatar.position, localOnly: true, volume: 0.8 }); var timestamp; var index = -1; if (jointDataToRecord[0]) { var dataToSave = { timestamp: false, data: [] }; leftHandRecorder.stopRecording(); timestamp = leftHandRecorder.recordingStartTimeMS; dataToSave.timestamp = timestamp; dataToSave.data = leftHandRecorder.currentPoseFrameData; leftHandRecorder.previouslyRecordedFrameData.push(dataToSave); leftHandRecorder.currentPoseFrameData = []; index = leftHandRecorder.previouslyRecordedFrameData.length - 1; } else { leftHandRecorder.previouslyRecordedFrameData.push("NODATA"); } if (jointDataToRecord[1]) { var dataToSave2 = { timestamp: false, data: [] }; rightHandRecorder.stopRecording(); timestamp = rightHandRecorder.recordingStartTimeMS; dataToSave2.timestamp = timestamp; dataToSave2.data = rightHandRecorder.currentPoseFrameData; rightHandRecorder.previouslyRecordedFrameData.push(dataToSave2); rightHandRecorder.currentPoseFrameData = []; index = rightHandRecorder.previouslyRecordedFrameData.length - 1; } else { rightHandRecorder.previouslyRecordedFrameData.push("NODATA"); } if (jointDataToRecord[2]) { var dataToSave3 = { timestamp: false, data: [] }; headRecorder.stopRecording(); timestamp = headRecorder.recordingStartTimeMS; dataToSave3.timestamp = timestamp; dataToSave3.data = headRecorder.currentPoseFrameData; headRecorder.previouslyRecordedFrameData.push(dataToSave3); headRecorder.currentPoseFrameData = []; index = headRecorder.previouslyRecordedFrameData.length - 1; } else { headRecorder.previouslyRecordedFrameData.push("NODATA"); } timestamp = new Date(timestamp); sendToQml({ method: 'appendRecordedGesture', index: index, recordedTime: timestamp.getHours() + ":" + timestamp.getMinutes() + ":" + timestamp.getSeconds(), recordedJoints: [jointDataToRecord[0], jointDataToRecord[1], jointDataToRecord[2]] }); updateGestureDetectionSystem(); } var isRecordingSelectedGestures = false; function toggleRecordingGesture() { if (!isRecordingSelectedGestures) { startRecordingSelectedGestures(); } else { stopRecordingSelectedGestures(); } } /******************************** // END Global Capture Start/Stop/Update functions ********************************/ /******************************** // START Controller Mapping ********************************/ var gesturesControllerMapping = false; var gesturesControllerMappingName = 'Hifi-Gestures-Mapping'; function maybeRegisterButtonMappings() { // Don't re-register if (gesturesControllerMapping) { return; } gesturesControllerMapping = Controller.newMapping(gesturesControllerMappingName); if (controllerType === "OculusTouch") { gesturesControllerMapping.from(Controller.Standard.RS).to(function (value) { if (value === 1.0) { toggleRecordingGesture(); } return; }); } else if (controllerType === "Vive") { gesturesControllerMapping.from(Controller.Standard.RightPrimaryThumb).to(function (value) { if (value === 1.0) { toggleRecordingGesture(); } return; }); } gesturesControllerMapping.enable(); } function disableButtonMappings() { if (gesturesControllerMapping) { gesturesControllerMapping.disable(); gesturesControllerMapping = false; } } /******************************** // END Controller Mapping ********************************/ /******************************** // START App-Related Functions ********************************/ // Function Name: sendToQml() // // Description: // -Use this function to send a message to the app's QML (i.e. to change appearances). The "message" argument is what is sent to // the app's QML in the format "{method, params}", like json-rpc. See also fromQml(). function sendToQml(message) { ui.sendMessage(message); } // Function Name: fromQml() // // Description: // -Called when a message is received from the app QML. The "message" argument is what is sent from the app QML // in the format "{method, params}", like json-rpc. See also sendToQml(). function fromQml(message) { switch (message.method) { case 'enableRecordingSwitchChanged': if (message.status) { maybeRegisterButtonMappings(); } else { disableButtonMappings(); } Settings.setValue('gestures/enableControllerMapping', message.status); break; case 'updateJointSelections': jointDataToRecord = message.selections; Settings.setValue('gestures/captureLeftHandData', jointDataToRecord[0]); Settings.setValue('gestures/captureRightHandData', jointDataToRecord[1]); Settings.setValue('gestures/captureHeadData', jointDataToRecord[2]); break; case 'toggleManualDataCapture': toggleRecordingGesture(); break; case 'modifyListeningForGestureIndex': // console.log("the gesture index is set to " + message.listeningForGestureIndex); gestureToDetect_index = message.listeningForGestureIndex; updateGestureDetectionSystem(); break; case 'clearRecordedGestures': gestureToDetect_index = -1; leftHandRecorder.previouslyRecordedFrameData = []; rightHandRecorder.previouslyRecordedFrameData = []; headRecorder.previouslyRecordedFrameData = []; break; case 'updateSensitivity': gestureDetectionSensitivityMultiplier = message.sensitivity; Settings.setValue('gestures/sensitivity', gestureDetectionSensitivityMultiplier); break; case 'updateMaxIgnoreDistance': avatarIgnoreMaxDistance = message.distance; Settings.setValue('gestures/maxIgnoreDistance', avatarIgnoreMaxDistance); maybePrint("ZRF: Updating max avatar ignore distance to '" + avatarIgnoreMaxDistance, DEBUG_IMPORTANT); break; case 'copyDataToClipboard': var dataToCopy = []; var leftHandData = "NODATA"; var rightHandData = "NODATA"; var headData = "NODATA"; if (leftHandRecorder.previouslyRecordedFrameData[message.index] !== "NODATA") { leftHandData = leftHandRecorder.previouslyRecordedFrameData[message.index]; } if (rightHandRecorder.previouslyRecordedFrameData[message.index] !== "NODATA") { rightHandData = rightHandRecorder.previouslyRecordedFrameData[message.index]; } if (headRecorder.previouslyRecordedFrameData[message.index] !== "NODATA") { headData = headRecorder.previouslyRecordedFrameData[message.index]; } dataToCopy.push(leftHandData); dataToCopy.push(rightHandData); dataToCopy.push(headData); Window.copyToClipboard(JSON.stringify(dataToCopy)); break; case 'importGestureData': var data = JSON.parse(message.data); if (data.length !== 3) { maybePrint('Unrecognized import data format, bailing!', DEBUG_URGENT); return; } leftHandRecorder.previouslyRecordedFrameData.push(data[0]); rightHandRecorder.previouslyRecordedFrameData.push(data[1]); headRecorder.previouslyRecordedFrameData.push(data[2]); appendGestureToQMLWithIndex(rightHandRecorder.previouslyRecordedFrameData.length - 1); break; default: maybePrint('Unrecognized message from Gestures.qml: ' + JSON.stringify(message), DEBUG_URGENT); } } function appendGestureToQMLWithIndex(index) { var timestamp; var recordedJoints = [false, false, false]; if (leftHandRecorder.previouslyRecordedFrameData[index] !== "NODATA") { recordedJoints[0] = true; timestamp = leftHandRecorder.previouslyRecordedFrameData[index].timestamp; } if (rightHandRecorder.previouslyRecordedFrameData[index] !== "NODATA") { recordedJoints[1] = true; timestamp = rightHandRecorder.previouslyRecordedFrameData[index].timestamp; } if (headRecorder.previouslyRecordedFrameData[index] !== "NODATA") { recordedJoints[2] = true; timestamp = headRecorder.previouslyRecordedFrameData[index].timestamp; } timestamp = new Date(timestamp); sendToQml({ method: 'appendRecordedGesture', index: index, recordedTime: timestamp.getHours() + ":" + timestamp.getMinutes() + ":" + timestamp.getSeconds(), recordedJoints: recordedJoints }); } function appendAllGesturesToQML() { for (var i = 0; i < leftHandRecorder.previouslyRecordedFrameData.length; i++) { appendGestureToQMLWithIndex(i); } } // Function Name: appUiOpened() // // Description: // - Called when the app's UI is opened // var APP_INITIALIZE_UI_DELAY = 500; // MS function appUiOpened() { // In the case of a remote QML app, it takes a bit of time // for the event bridge to actually connect, so we have to wait... Script.setTimeout(function () { sendToQml({ method: 'initializeUI', masterSwitchOn: !!gesturesControllerMapping, jointDataToRecord: jointDataToRecord, currentlyDetectingGesture: gestureToDetect_index, gestureDetectionSensitivityMultiplier: gestureDetectionSensitivityMultiplier, maxIgnoreDistance: avatarIgnoreMaxDistance }); appendAllGesturesToQML(); }, APP_INITIALIZE_UI_DELAY); } // Function Name: appUiClosed() // // Description: // - Called when the app's UI is closed // function appUiClosed() { } // Function Name: startup() // // Description: // -startup() will be called when the script is loaded. // var ui; var controllerType = "Other"; var avatarIgnoreMaxDistance = 5.0; function startup() { ui = new AppUi({ buttonName: "GESTURES", home: Script.resolvePath('https://hifi-content.s3.amazonaws.com/zfox/gestureApp/Gestures.qml'), onOpened: appUiOpened, onClosed: appUiClosed, onMessage: fromQml, sortOrder: 15, normalButton: Script.resourcesPath() + "icons/tablet-icons/avatar-record-i.svg", activeButton: Script.resourcesPath() + "icons/tablet-icons/avatar-record-a.svg" }); // Controller type detection var VRDevices = Controller.getDeviceNames().toString(); if (VRDevices) { if (VRDevices.indexOf("Vive") !== -1) { controllerType = "Vive"; } else if (VRDevices.indexOf("OculusTouch") !== -1) { controllerType = "OculusTouch"; } } if (Settings.getValue('gestures/enableControllerMapping', false)) { maybeRegisterButtonMappings(); } avatarIgnoreMaxDistance = Settings.getValue('gestures/maxIgnoreDistance', 5.0); var borderModelProperties = { position: getPositionForIcon(), lifetime: 100, modelURL: "file:///c:/angus/javascript_bak/gesture/outline.fbx", name: " border", type : "Model" }; var loadingBarModelProperties = { position: getPositionForIcon(), lifetime: 100, dimensions: { x: 0.001, y: 0.1, z: 0.1 }, modelURL: "file:///c:/angus/javascript_bak/gesture/loadingBar.fbx", name: " border", type : "Model" }; //loadingBarEntityID = Entities.addEntity(loadingBarModelProperties); //borderEntityID = Entities.addEntity(borderModelProperties); //Entities.editEntity(borderEntityID, {dimensions: { x: 0.001, y: 0.1, z: 0.5 }}); } // Function Name: shutdown() // // Description: // - Called when the script ends (i.e. is stopped). // function shutdown() { appUiClosed(); //Entities.deleteEntity(borderEntityID); //Entities.deleteEntity(loadingBarEntityID); Overlays.deleteOverlay(animShush.uuid); Overlays.deleteOverlay(animShush.border); Overlays.deleteOverlay(animShush.loadBar); Overlays.deleteOverlay(animShush.textUUID); DebugDraw.removeMarker("head"); DebugDraw.removeMarker("pos1"); DebugDraw.removeMarker("pos2"); DebugDraw.removeMarker("pos3"); DebugDraw.removeMarker("pos4"); DebugDraw.removeMarker("pos5"); DebugDraw.removeMarker("pos6"); } var SOUND_GESTURE_RECORDING_START = SoundCache.getSound(Script.resolvePath("https://hifi-content.s3.amazonaws.com/zfox/gestureApp/startRecording.wav")); var SOUND_GESTURE_RECORDING_STOP = SoundCache.getSound(Script.resolvePath("https://hifi-content.s3.amazonaws.com/zfox/gestureApp/stopRecording.wav")); var SOUND_GESTURE_DETECTED = SoundCache.getSound(Script.resolvePath("https://hifi-content.s3.amazonaws.com/zfox/gestureApp/gestureDetected.wav")); startup(); Script.scriptEnding.connect(shutdown); /******************************** // END App-Related Functions ********************************/ }()); // END LOCAL_SCOPE