mirror of
https://github.com/overte-org/community-apps.git
synced 2025-04-05 21:22:00 +02:00
1024 lines
34 KiB
HTML
1024 lines
34 KiB
HTML
<!--
|
|
index.html
|
|
|
|
Created by George Deac, October 21st, 2023.
|
|
Copyright 2023 George Deac.
|
|
Copyright 2023 The MediaPipe Authors.
|
|
Copyright 2024, Overte e.V.
|
|
|
|
Overte Application for Mediapipe face tracking in Desktop mode.
|
|
|
|
Distributed under the Apache License, Version 2.0.
|
|
See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
|
-->
|
|
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<title>MediaPipe Face Landmarker</title>
|
|
<style>
|
|
@use "@material";
|
|
body {
|
|
font-family: helvetica, arial, sans-serif;
|
|
margin: 2em;
|
|
bgcolor: "white";
|
|
color: #3d3d3d;
|
|
--mdc-theme-primary: #007f8b;
|
|
--mdc-theme-on-primary: #f1f3f4;
|
|
}
|
|
|
|
h1 {
|
|
font-style: italic;
|
|
color: #ff6f00;
|
|
color: #007f8b;
|
|
}
|
|
|
|
h2 {
|
|
clear: both;
|
|
}
|
|
|
|
em {
|
|
font-weight: bold;
|
|
}
|
|
|
|
video {
|
|
clear: both;
|
|
display: block;
|
|
transform: rotateY(180deg);
|
|
-webkit-transform: rotateY(180deg);
|
|
-moz-transform: rotateY(180deg);
|
|
}
|
|
|
|
section {
|
|
opacity: 1;
|
|
transition: opacity 500ms ease-in-out;
|
|
}
|
|
|
|
header,
|
|
footer {
|
|
clear: both;
|
|
}
|
|
|
|
.removed {
|
|
display: none;
|
|
}
|
|
|
|
.invisible {
|
|
opacity: 0.2;
|
|
}
|
|
|
|
.note {
|
|
font-style: italic;
|
|
font-size: 130%;
|
|
}
|
|
|
|
.videoView,
|
|
.detectOnClick,
|
|
.blend-shapes {
|
|
position: relative;
|
|
|
|
width: 48%;
|
|
margin: 2% 1%;
|
|
cursor: pointer;
|
|
}
|
|
|
|
.videoView p,
|
|
.detectOnClick p {
|
|
position: absolute;
|
|
padding: 5px;
|
|
background-color: #007f8b;
|
|
color: #fff;
|
|
border: 1px dashed rgba(255, 255, 255, 0.7);
|
|
z-index: 2;
|
|
font-size: 12px;
|
|
margin: 0;
|
|
}
|
|
|
|
.highlighter {
|
|
background: rgba(0, 255, 0, 0.25);
|
|
border: 1px dashed #fff;
|
|
z-index: 1;
|
|
position: absolute;
|
|
}
|
|
|
|
.canvas {
|
|
z-index: 1;
|
|
position: absolute;
|
|
pointer-events: none;
|
|
}
|
|
|
|
.output_canvas {
|
|
transform: rotateY(180deg);
|
|
-webkit-transform: rotateY(180deg);
|
|
-moz-transform: rotateY(180deg);
|
|
}
|
|
|
|
.detectOnClick {
|
|
z-index: 0;
|
|
}
|
|
|
|
.detectOnClick img {
|
|
width: 100%;
|
|
}
|
|
|
|
.blend-shapes-item {
|
|
display: flex;
|
|
align-items: center;
|
|
height: 20px;
|
|
}
|
|
|
|
.blend-shapes-label {
|
|
display: flex;
|
|
width: 120px;
|
|
justify-content: flex-end;
|
|
align-items: center;
|
|
margin-right: 4px;
|
|
}
|
|
|
|
.blend-shapes-value {
|
|
display: flex;
|
|
height: 16px;
|
|
align-items: center;
|
|
background-color: #007f8b;
|
|
}
|
|
|
|
.wrapper {
|
|
max-width: 800px;
|
|
margin: 50px auto;
|
|
}
|
|
|
|
h1 {
|
|
font-weight: 200;
|
|
font-size: 3em;
|
|
margin: 0 0 0.1em 0;
|
|
}
|
|
|
|
h2 {
|
|
font-weight: 200;
|
|
font-size: 0.9em;
|
|
margin: 0 0 50px;
|
|
color: #555;
|
|
}
|
|
|
|
a {
|
|
margin-top: 50px;
|
|
display: block;
|
|
color: #3e95cd;
|
|
}
|
|
|
|
.switch {
|
|
position: relative;
|
|
display: inline-block;
|
|
width: 60px;
|
|
height: 34px;
|
|
}
|
|
|
|
/* Hide default HTML checkbox */
|
|
.switch input {
|
|
opacity: 0;
|
|
width: 0;
|
|
height: 0;
|
|
}
|
|
|
|
/* The slider */
|
|
.slider {
|
|
position: absolute;
|
|
cursor: pointer;
|
|
top: 0;
|
|
left: 0;
|
|
right: 0;
|
|
bottom: 0;
|
|
background-color: #ccc;
|
|
-webkit-transition: .4s;
|
|
transition: .4s;
|
|
}
|
|
|
|
.slider:before {
|
|
position: absolute;
|
|
content: "";
|
|
height: 26px;
|
|
width: 26px;
|
|
left: 4px;
|
|
bottom: 4px;
|
|
background-color: white;
|
|
-webkit-transition: .4s;
|
|
transition: .4s;
|
|
}
|
|
|
|
input:checked + .slider {
|
|
background-color: #007db8;
|
|
}
|
|
|
|
input:focus + .slider {
|
|
box-shadow: 0 0 1px #007db8;
|
|
}
|
|
|
|
input:checked + .slider:before {
|
|
-webkit-transform: translateX(26px);
|
|
-ms-transform: translateX(26px);
|
|
transform: translateX(26px);
|
|
}
|
|
|
|
/* Rounded sliders */
|
|
.slider.round {
|
|
border-radius: 34px;
|
|
}
|
|
|
|
.slider.round:before {
|
|
border-radius: 50%;
|
|
}
|
|
</style>
|
|
|
|
<script>
|
|
window.console = window.console || function (t) {
|
|
};
|
|
</script>
|
|
|
|
<meta charset="utf-8">
|
|
<meta http-equiv="Cache-control" content="no-cache, no-store, must-revalidate">
|
|
<meta http-equiv="Pragma" content="no-cache">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
|
|
|
<link href="https://unpkg.com/material-components-web@latest/dist/material-components-web.min.css" rel="stylesheet">
|
|
<script src="https://unpkg.com/material-components-web@latest/dist/material-components-web.min.js"></script>
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.3/dist/Chart.min.js"></script>
|
|
</head>
|
|
<body bgcolor="white">
|
|
|
|
<div id="liveView" class="videoView">
|
|
<table width="320px">
|
|
<tr>
|
|
<td>
|
|
<button id="webcamButton" style="white-space:nowrap;width:100px" class="mdc-button mdc-button--raised">
|
|
<span class="mdc-button__ripple"></span>
|
|
<span class="mdc-button__label">VIDEO ON</span>
|
|
</button>
|
|
</td>
|
|
<td>
|
|
|
|
</td>
|
|
<td>
|
|
Use Head Navigation:
|
|
</td>
|
|
<td>
|
|
<label class="switch">
|
|
<input type="checkbox" id="switchmove" class="switchmove">
|
|
<span class="slider"></span>
|
|
</label>
|
|
</td>
|
|
<td>
|
|
Show preview window:
|
|
</td>
|
|
<td>
|
|
<label class="switch">
|
|
<input type="checkbox" id="enable_webcam_display" class="switchmove">
|
|
<span class="slider"></span>
|
|
</label>
|
|
</td>
|
|
</tr>
|
|
<tr><td colspan="4">
|
|
<div class="select">
|
|
<select id="videoSource"></select>
|
|
</div>
|
|
</td></tr>
|
|
</table><br>
|
|
<div id="preview_camera_area" style="display:none; position: relative;">
|
|
<video id="webcam" autoplay playsinline></video>
|
|
<canvas class="output_canvas" id="output_canvas" style="position: absolute; left: 0px; top: 0px;"></canvas>
|
|
</div>
|
|
</div>
|
|
|
|
<select id="landmarkDropdown" onchange="handleLandmarkChange()">
|
|
<!-- Options for the dropdown will be populated in the script -->
|
|
</select>
|
|
</div>
|
|
|
|
<div>
|
|
<input type="text" id="maxY" value="100"/>
|
|
<button onclick="increaseY()">Increase</button>
|
|
<button onclick="decreaseY()">Decrease</button>
|
|
</div>
|
|
|
|
|
|
<!-- canvas element in a container -->
|
|
<div class="wrapper">
|
|
<canvas id="canvas" width="480" height="480"></canvas>
|
|
</div>
|
|
|
|
<p id="output"></p>
|
|
<div class="blend-shapes">
|
|
<ul class="blend-shapes-list" id="video-blend-shapes"></ul>
|
|
</div>
|
|
</section>
|
|
<!--/body>
|
|
</html-->
|
|
|
|
<script src="jquery.min.js"></script>
|
|
<script>
|
|
|
|
var channel = "org.overte.application.emocam";
|
|
|
|
// Graph max range adjust
|
|
var maxY = 100; // Default Y max value
|
|
|
|
function increaseY() {
|
|
if (maxY < 100) {
|
|
maxY += 25;
|
|
} else maxY += 100;
|
|
document.getElementById('maxY').value = maxY;
|
|
updateChartMaxY();
|
|
}
|
|
|
|
function decreaseY() {
|
|
if (maxY <= 100) {
|
|
maxY -= 25;
|
|
} else maxY -= 100;
|
|
document.getElementById('maxY').value = maxY;
|
|
updateChartMaxY();
|
|
}
|
|
|
|
function updateChartMaxY() {
|
|
|
|
var dropdown = document.getElementById("landmarkDropdown");
|
|
var selectedLandmark = dropdown.options[dropdown.selectedIndex].value;
|
|
|
|
// update the data for the chart
|
|
var mapping = landmarkMappings[selectedLandmark];
|
|
|
|
// Clip maxY logic here for mapping.y_data_1 before updating chart
|
|
for (var i = 0; i < mapping.y_data_1.length; i++) {
|
|
if (mapping.y_data_1[i] > maxY) {
|
|
mapping.y_data_1[i] = maxY;
|
|
}
|
|
}
|
|
|
|
window.myChart.data.labels = mapping.x_data;
|
|
window.myChart.data.datasets[0].data = mapping.y_data_1;
|
|
// update label for the line to reflect the currently selected landmark
|
|
window.myChart.data.datasets[0].label = selectedLandmark + " morph mapping";
|
|
|
|
window.myChart.options.scales.yAxes[0].ticks.max = maxY;
|
|
|
|
window.myChart.update();
|
|
}
|
|
|
|
function generateControlPoints(numPoints) {
|
|
var x_data = [];
|
|
var y_data_1 = [];
|
|
|
|
var stepSize = 100 / (numPoints - 1); // determine the step size based on the desired number of points
|
|
|
|
for (var i = 0; i < numPoints; i++) {
|
|
var val = stepSize * i;
|
|
x_data.push(val);
|
|
y_data_1.push(val);
|
|
}
|
|
|
|
return {x_data, y_data_1};
|
|
}
|
|
|
|
|
|
// Initialize landmarks
|
|
var landmarks = [
|
|
"browDownLeft",
|
|
"browDownRight",
|
|
"browInnerUp",
|
|
"browOuterUpLeft",
|
|
"browOuterUpRight",
|
|
"cheekPuff",
|
|
"cheekSquintLeft",
|
|
"cheekSquintRight",
|
|
"eyeBlinkLeft",
|
|
"eyeBlinkRight",
|
|
"eyeLookDownLeft",
|
|
"eyeLookDownRight",
|
|
"eyeLookInLeft",
|
|
"eyeLookInRight",
|
|
"eyeLookOutLeft",
|
|
"eyeLookOutRight",
|
|
"eyeLookUpLeft",
|
|
"eyeLookUpRight",
|
|
"eyeSquintLeft",
|
|
"eyeSquintRight",
|
|
"eyeWideLeft",
|
|
"eyeWideRight",
|
|
"jawForward",
|
|
"jawLeft",
|
|
"jawOpen",
|
|
"jawRight",
|
|
"mouthClose",
|
|
"mouthDimpleLeft",
|
|
"mouthDimpleRight",
|
|
"mouthFrownLeft",
|
|
"mouthFrownRight",
|
|
"mouthFunnel",
|
|
"mouthLeft",
|
|
"mouthLowerDownLeft",
|
|
"mouthLowerDownRight",
|
|
"mouthPressLeft",
|
|
"mouthPressRight",
|
|
"mouthPucker",
|
|
"mouthRight",
|
|
"mouthRollLower",
|
|
"mouthRollUpper",
|
|
"mouthShrugLower",
|
|
"mouthShrugUpper",
|
|
"mouthSmileLeft",
|
|
"mouthSmileRight",
|
|
"mouthStretchLeft",
|
|
"mouthStretchRight",
|
|
"mouthUpperUpLeft",
|
|
"mouthUpperUpRight",
|
|
"noseSneerLeft",
|
|
"noseSneerRight"
|
|
];
|
|
|
|
var landmarkMappings = {};
|
|
var testInput = 50;
|
|
var activePoint = null;
|
|
|
|
// Generate default linear mappings for each landmark morph with 6 control points each
|
|
for (var i = 0; i < landmarks.length; i++) {
|
|
landmarkMappings[landmarks[i]] = generateControlPoints(5 + 1);
|
|
}
|
|
|
|
// Handle dropdown change
|
|
function handleLandmarkChange() {
|
|
var dropdown = document.getElementById("landmarkDropdown");
|
|
var selectedLandmark = dropdown.options[dropdown.selectedIndex].value;
|
|
|
|
// update the data for the chart
|
|
var mapping = landmarkMappings[selectedLandmark];
|
|
|
|
// Clip maxY logic here for mapping.y_data_1 before updating chart
|
|
for (var i = 0; i < mapping.y_data_1.length; i++) {
|
|
if (mapping.y_data_1[i] > maxY) {
|
|
mapping.y_data_1[i] = maxY;
|
|
}
|
|
}
|
|
|
|
window.myChart.data.labels = mapping.x_data;
|
|
window.myChart.data.datasets[0].data = mapping.y_data_1;
|
|
// update label for the line to reflect the currently selected landmark
|
|
window.myChart.data.datasets[0].label = selectedLandmark + " morph mapping";
|
|
|
|
|
|
window.myChart.update();
|
|
|
|
}
|
|
|
|
// Functions to handle pointer events
|
|
function down_handler(event) {
|
|
// check for data point near event location
|
|
const points = window.myChart.getElementAtEvent(event, {intersect: false});
|
|
if (points.length > 0) {
|
|
// grab nearest point, start dragging
|
|
activePoint = points[0];
|
|
canvas.onpointermove = move_handler;
|
|
}
|
|
;
|
|
};
|
|
|
|
function up_handler(event) {
|
|
// release grabbed point, stop dragging
|
|
activePoint = null;
|
|
canvas.onpointermove = null;
|
|
};
|
|
|
|
function move_handler(event) {
|
|
// locate grabbed point in chart data
|
|
if (activePoint != null) {
|
|
var data = activePoint._chart.data;
|
|
var datasetIndex = activePoint._datasetIndex;
|
|
|
|
// read mouse position
|
|
const helpers = Chart.helpers;
|
|
var position = helpers.getRelativePosition(event, myChart);
|
|
|
|
// convert mouse position to chart y axis value
|
|
var chartArea = window.myChart.chartArea;
|
|
var yAxis = window.myChart.scales["y-axis-0"];
|
|
var yValue = map(position.y, chartArea.bottom, chartArea.top, yAxis.min, yAxis.max);
|
|
|
|
// Prevent yValue from exceeding bounds
|
|
yValue = Math.min(yValue, maxY);
|
|
yValue = Math.max(yValue, 0);
|
|
|
|
// update y value of active data point
|
|
data.datasets[datasetIndex].data[activePoint._index] = yValue;
|
|
window.myChart.update();
|
|
}
|
|
;
|
|
};
|
|
|
|
// Draw a line chart on the canvas context
|
|
window.onload = function () {
|
|
var ctx = document.getElementById("canvas").getContext("2d");
|
|
var canvas = document.getElementById("canvas");
|
|
var mapping = landmarkMappings[landmarks[0]]; // start with first landmark
|
|
window.myChart = Chart.Line(ctx, {
|
|
data: {
|
|
labels: mapping.x_data,
|
|
datasets: [
|
|
{
|
|
data: mapping.y_data_1,
|
|
label: landmarks[0] + " morph mapping",
|
|
borderColor: "#3e95cd",
|
|
fill: false,
|
|
pointRadius: 8, // Adjust this for point size
|
|
pointHoverRadius: 10, // Adjust this for point size on hover
|
|
},
|
|
]
|
|
},
|
|
options: {
|
|
animation: {
|
|
duration: 0
|
|
},
|
|
tooltips: {
|
|
mode: 'nearest'
|
|
},
|
|
scales: {
|
|
yAxes: [{
|
|
ticks: {
|
|
min: 0,
|
|
max: maxY
|
|
}
|
|
}]
|
|
}
|
|
}
|
|
});
|
|
|
|
// set pointer event handlers for canvas element
|
|
canvas.onpointerdown = down_handler;
|
|
canvas.onpointerup = up_handler;
|
|
canvas.onpointermove = null;
|
|
|
|
// Populate dropdown with landmarks
|
|
var dropdown = document.getElementById("landmarkDropdown");
|
|
for (var i = 0; i < landmarks.length; i++) {
|
|
var option = document.createElement("option");
|
|
option.text = landmarks[i];
|
|
option.value = landmarks[i];
|
|
dropdown.add(option);
|
|
}
|
|
|
|
};
|
|
|
|
|
|
// MinMax map value to other coordinate system
|
|
function map(value, start1, stop1, start2, stop2) {
|
|
return start2 + (stop2 - start2) * ((value - start1) / (stop1 - start1))
|
|
};
|
|
|
|
|
|
function monotoneCubicSplineInterpolation(x, y) {
|
|
var size = x.length;
|
|
var delta = [];
|
|
for (var i = 0; i < size - 1; i++) {
|
|
delta[i] = (y[i + 1] - y[i]) / (x[i + 1] - x[i]);
|
|
}
|
|
|
|
var m = [];
|
|
m[0] = delta[0];
|
|
for (var i = 1; i < size - 1; i++) {
|
|
m[i] = (delta[i - 1] + delta[i]) * 0.5;
|
|
}
|
|
m[size - 1] = delta[size - 2];
|
|
|
|
for (var i = 0; i < size - 1; i++) {
|
|
if (delta[i] === 0) {
|
|
m[i] = m[i + 1] = 0;
|
|
} else {
|
|
var alpha = m[i] / delta[i];
|
|
var beta = m[i + 1] / delta[i];
|
|
if (alpha * alpha + beta * beta > 9) {
|
|
var tau = 3 / Math.sqrt(alpha * alpha + beta * beta);
|
|
m[i] = tau * alpha * delta[i];
|
|
m[i + 1] = tau * beta * delta[i];
|
|
}
|
|
}
|
|
}
|
|
|
|
return function (val) {
|
|
if (val < x[0] || val > x[size - 1]) {
|
|
throw new Error("The input value is out of the data range.");
|
|
}
|
|
var i = 1;
|
|
while (x[i] < val) i++;
|
|
i--;
|
|
|
|
var h = (x[i + 1] - x[i]);
|
|
var t = (val - x[i]) / h;
|
|
return (y[i] * (1 + 2 * t) + h * m[i] * t) * (1 - t) * (1 - t) +
|
|
(y[i + 1] * (3 - 2 * t) + h * m[i + 1] * (t - 1)) * t * t;
|
|
};
|
|
}
|
|
|
|
|
|
function applyMapping(value, x_data, y_data_1) {
|
|
var cubicSpline = monotoneCubicSplineInterpolation(x_data, y_data_1);
|
|
var yValue = cubicSpline(value);
|
|
return yValue;
|
|
}
|
|
|
|
|
|
function applyLandmarksMapping(values) {
|
|
// Object to hold the mapped values
|
|
var mappedValues = {};
|
|
|
|
// Loop through each key-value pair in the provided JSON object
|
|
for (var landmark in values) {
|
|
// Skip the neutral landmark
|
|
if (landmark === "_neutral") {
|
|
continue;
|
|
}
|
|
if (values.hasOwnProperty(landmark)) {
|
|
// Check if there is a mapping for this landmark
|
|
if (landmarkMappings.hasOwnProperty(landmark)) {
|
|
// Apply the mapping
|
|
var value = values[landmark];
|
|
value *= 100;
|
|
var mapping = landmarkMappings[landmark];
|
|
var mappedValue = monotoneCubicSplineInterpolation(mapping.x_data, mapping.y_data_1)(value);
|
|
mappedValue /= 100;
|
|
|
|
// Store the mapped value
|
|
mappedValues[landmark] = mappedValue;
|
|
}
|
|
}
|
|
}
|
|
|
|
return mappedValues;
|
|
}
|
|
|
|
|
|
</script>
|
|
<script id="rendered-js" type="module">
|
|
// Copyright 2023 The MediaPipe Authors.
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
import vision from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0";
|
|
|
|
const {FaceLandmarker, FilesetResolver, DrawingUtils} = vision;
|
|
const demosSection = document.getElementById("demos");
|
|
const videoBlendShapes = document.getElementById("video-blend-shapes");
|
|
const enableWebcamDisplaySwitch = document.querySelector('#enable_webcam_display')
|
|
let faceLandmarker;
|
|
let runningMode = "IMAGE";
|
|
let enableWebcamButton;
|
|
let webcamRunning = false;
|
|
const videoWidth = 400;
|
|
var checkedValue = null;
|
|
var yawDegrees;
|
|
var pitchDegrees;
|
|
var rollDegrees;
|
|
var calibration = 0;
|
|
var yawC = 0;
|
|
var pitchC = 0;
|
|
var rollC = 0;
|
|
// Before we can use HandLandmarker class we must wait for it to finish
|
|
// loading. Machine Learning models can be large and take a moment to
|
|
// get everything needed to run.
|
|
async function runDemo() {
|
|
// Read more `CopyWebpackPlugin`, copy wasm set from "https://cdn.skypack.dev/node_modules" to `/wasm`
|
|
const filesetResolver = await FilesetResolver.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm");
|
|
faceLandmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
|
|
baseOptions: {
|
|
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
|
|
delegate: "GPU"
|
|
},
|
|
outputFaceBlendshapes: true,
|
|
outputFacialTransformationMatrixes: true,
|
|
runningMode,
|
|
numFaces: 1
|
|
});
|
|
//demosSection.classList.remove("invisible");
|
|
}
|
|
|
|
runDemo();
|
|
|
|
enableWebcamDisplaySwitch.addEventListener('change', (event) => {
|
|
let enabled = event.target.checked;
|
|
preview_camera_area.style.display = enabled ? "block" : "none";
|
|
})
|
|
|
|
const video = document.getElementById("webcam");
|
|
const canvasElement = document.getElementById("output_canvas");
|
|
const canvasCtx = canvasElement.getContext("2d");
|
|
|
|
// Check if webcam access is supported.
|
|
function hasGetUserMedia() {
|
|
return !!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia);
|
|
}
|
|
|
|
// If webcam supported, add event listener to button for when user
|
|
// wants to activate it.
|
|
if (hasGetUserMedia()) {
|
|
enableWebcamButton = document.getElementById("webcamButton");
|
|
enableWebcamButton.addEventListener("click", enableCam);
|
|
} else {
|
|
console.warn("getUserMedia() is not supported by your browser");
|
|
}
|
|
$(document).on("click", ".vid_link", function (event) {
|
|
var videoElement = document.querySelector('video');
|
|
hrefValue = $(this).text();
|
|
var idValue = $(this).attr('id');
|
|
vidopttrigger = true;
|
|
document.querySelector("#videoSource").value=idValue;
|
|
$("#videoSource").val(idValue).prop('selected', true);
|
|
$("#videoSource").val(idValue).change();
|
|
$("#videoDropdown").val(hrefValue);
|
|
});
|
|
// Enable the live webcam view and start detection.
|
|
function enableCam(event) {
|
|
var videoElement = document.querySelector('video');
|
|
var videoSelect = document.querySelector('select#videoSource');
|
|
videoSelect.onchange = getStream;
|
|
if (!faceLandmarker) {
|
|
console.log("Wait! faceLandmarker not loaded yet.");
|
|
return;
|
|
}
|
|
if (webcamRunning === true) {
|
|
webcamRunning = false;
|
|
enableWebcamButton.innerText = "VIDEO ON";
|
|
$('#videoSource').empty();
|
|
window.stream.getTracks().forEach(track => {
|
|
track.stop();
|
|
});
|
|
} else {
|
|
getStream().then(getDevices).then(gotDevices);
|
|
webcamRunning = true;
|
|
enableWebcamButton.innerText = "VIDEO OFF";
|
|
video.addEventListener("loadeddata", predictWebcam);
|
|
}
|
|
// getUsermedia parameters.
|
|
// Copyright 2017 Google Inc.
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
function getDevices() {
|
|
// AFAICT in Safari this only gets default devices until gUM is called :/
|
|
return navigator.mediaDevices.enumerateDevices();
|
|
}
|
|
|
|
function gotDevices(deviceInfos) {
|
|
window.deviceInfos = deviceInfos; // make available to console
|
|
console.log('Available input and output devices:', deviceInfos);
|
|
for (const deviceInfo of deviceInfos) {
|
|
const option = document.createElement('option');
|
|
option.value = deviceInfo.deviceId;
|
|
if (deviceInfo.kind === 'videoinput') {
|
|
option.text = deviceInfo.label || `Camera ${videoSelect.length + 1}`;
|
|
videoSelect.appendChild(option);
|
|
$("#vid_list").append("<a class='vid_link' href='#' id='"+option.value+"'>" + option.text + "</a>");
|
|
}
|
|
}
|
|
}
|
|
|
|
function getStream() {
|
|
if (window.stream) {
|
|
window.stream.getTracks().forEach(track => {
|
|
track.stop();
|
|
});
|
|
}
|
|
|
|
const videoSource = videoSelect.value;
|
|
const constraints = {
|
|
video: {deviceId: videoSource ? {exact: videoSource} : undefined}
|
|
};
|
|
return navigator.mediaDevices.getUserMedia(constraints).
|
|
then(gotStream).catch(handleError);
|
|
}
|
|
|
|
function gotStream(stream) {
|
|
window.stream = stream; // make stream available to console
|
|
videoSelect.selectedIndex = [...videoSelect.options].
|
|
findIndex(option => option.text === stream.getVideoTracks()[0].label);
|
|
videoElement.srcObject = stream;
|
|
}
|
|
|
|
function handleError(error) {
|
|
console.error('Error: ', error);
|
|
}
|
|
}
|
|
|
|
let lastVideoTime = -1;
|
|
let results = undefined;
|
|
const drawingUtils = new DrawingUtils(canvasCtx);
|
|
|
|
async function predictWebcam() {
|
|
const radio = video.videoHeight / video.videoWidth;
|
|
video.style.width = videoWidth + "px";
|
|
video.style.height = videoWidth * radio + "px";
|
|
canvasElement.style.width = videoWidth + "px";
|
|
canvasElement.style.height = videoWidth * radio + "px";
|
|
canvasElement.width = video.videoWidth;
|
|
canvasElement.height = video.videoHeight;
|
|
// Now let's start detecting the stream.
|
|
if (runningMode === "IMAGE") {
|
|
runningMode = "LIVE_STREAM";
|
|
await faceLandmarker.setOptions({runningMode: runningMode});
|
|
}
|
|
let nowInMs = Date.now();
|
|
if (lastVideoTime !== video.currentTime) {
|
|
lastVideoTime = video.currentTime;
|
|
results = faceLandmarker.detectForVideo(video, nowInMs);
|
|
}
|
|
if (results.faceLandmarks && results.faceLandmarks.length > 0) {
|
|
|
|
// Define multiple reference points (ensure that they are distributed evenly across, 1 for the top face, 1 for the bottom face, 1 for the left face, 1 for the right face)
|
|
const referencePoints = [
|
|
results.faceLandmarks[0][10],
|
|
results.faceLandmarks[0][152],
|
|
results.faceLandmarks[0][33],
|
|
results.faceLandmarks[0][263],
|
|
// Add more reference points if desired...
|
|
];
|
|
|
|
// Filter out undefined or null points (if they are not detected by the model at that time)
|
|
const validPoints = referencePoints.filter(point => point !== undefined && point !== null);
|
|
|
|
// Check if there are enough valid points to proceed (we need at least 3 point to build a plane normal)
|
|
if (validPoints.length >= 3) {
|
|
// Calculate the centroid of the valid points
|
|
const centroid = validPoints.reduce((acc, point) => {
|
|
return {x: acc.x + point.x, y: acc.y + point.y, z: acc.z + point.z};
|
|
}, {x: 0, y: 0, z: 0});
|
|
|
|
centroid.x /= validPoints.length;
|
|
centroid.y /= validPoints.length;
|
|
centroid.z /= validPoints.length;
|
|
|
|
// Calculate the normal vector by averaging cross products
|
|
let normalVector = {x: 0, y: 0, z: 0};
|
|
for (let i = 0; i < validPoints.length - 1; i++) {
|
|
const vector1 = {
|
|
x: validPoints[i].x - centroid.x,
|
|
y: validPoints[i].y - centroid.y,
|
|
z: validPoints[i].z - centroid.z
|
|
};
|
|
const vector2 = {
|
|
x: validPoints[i + 1].x - centroid.x,
|
|
y: validPoints[i + 1].y - centroid.y,
|
|
z: validPoints[i + 1].z - centroid.z
|
|
};
|
|
|
|
const crossProduct = {
|
|
x: vector1.y * vector2.z - vector1.z * vector2.y,
|
|
y: vector1.z * vector2.x - vector1.x * vector2.z,
|
|
z: vector1.x * vector2.y - vector1.y * vector2.x
|
|
};
|
|
|
|
normalVector.x += crossProduct.x;
|
|
normalVector.y += crossProduct.y;
|
|
normalVector.z += crossProduct.z;
|
|
}
|
|
|
|
normalVector.x /= validPoints.length - 1;
|
|
normalVector.y /= validPoints.length - 1;
|
|
normalVector.z /= validPoints.length - 1;
|
|
|
|
// Normalize the normal vector (optional but can give better results)
|
|
const magnitude = Math.sqrt(normalVector.x ** 2 + normalVector.y ** 2 + normalVector.z ** 2);
|
|
const normalizedNormal = {
|
|
x: normalVector.x / magnitude,
|
|
y: normalVector.y / magnitude,
|
|
z: normalVector.z / magnitude
|
|
};
|
|
|
|
const sidewaysVector = {
|
|
x: referencePoints[2].x - referencePoints[3].x,
|
|
y: referencePoints[2].y - referencePoints[3].y,
|
|
z: referencePoints[2].z - referencePoints[3].z
|
|
};
|
|
|
|
const upwardVector = {
|
|
x: sidewaysVector.y * normalizedNormal.z - sidewaysVector.z * normalizedNormal.y,
|
|
y: sidewaysVector.z * normalizedNormal.x - sidewaysVector.x * normalizedNormal.z,
|
|
z: sidewaysVector.x * normalizedNormal.y - sidewaysVector.y * normalizedNormal.x
|
|
};
|
|
|
|
// Normalize the upward normal vector (optional but can give better results)
|
|
const upwardMagnitude = Math.sqrt(upwardVector.x ** 2 + upwardVector.y ** 2 + upwardVector.z ** 2);
|
|
const upwardNormal = {
|
|
x: upwardVector.x / upwardMagnitude,
|
|
y: upwardVector.y / upwardMagnitude,
|
|
z: upwardVector.z / upwardMagnitude
|
|
};
|
|
|
|
// Calculate the yaw, pitch, and roll angles
|
|
const yaw = Math.atan2(normalizedNormal.x, normalizedNormal.z);
|
|
const pitch = Math.atan2(-normalizedNormal.y, Math.sqrt(normalizedNormal.x ** 2 + normalizedNormal.z ** 2));
|
|
|
|
const planeRightZ = Math.sin(yaw);
|
|
const planeRightX = -Math.cos(yaw);
|
|
|
|
let roll = -Math.asin(upwardNormal.x * planeRightX + upwardNormal.z * planeRightZ);
|
|
|
|
if (upwardNormal.y < 0) {
|
|
roll = Math.sign(roll) * Math.PI - roll;
|
|
}
|
|
|
|
// Convert to degrees
|
|
if (calibration === 0) {
|
|
yawC = yaw * (180 / Math.PI);
|
|
pitchC = pitch * (180 / Math.PI);
|
|
rollC = roll * (180 / Math.PI);
|
|
//console.log("YawC: ", yawC, "PitchC: ", pitchC, "RollC: ", rollC);
|
|
calibration = 1;
|
|
}
|
|
yawDegrees = yaw * (180 / Math.PI) - yawC;
|
|
pitchDegrees = pitch * (180 / Math.PI) - pitchC;
|
|
rollDegrees = roll * (180 / Math.PI) - rollC;
|
|
|
|
//console.log("Yaw: ", yawDegrees, "Pitch: ", pitchDegrees, "Roll: ", rollDegrees);
|
|
}
|
|
|
|
//draw on canvas
|
|
let debugEnable = false;
|
|
if (debugEnable) {
|
|
for (const landmarks of results.faceLandmarks) {
|
|
drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_TESSELATION, {
|
|
color: "#C0C0C070",
|
|
lineWidth: 1
|
|
});
|
|
drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_RIGHT_EYE, {color: "#FF3030"});
|
|
drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_RIGHT_EYEBROW, {color: "#FF3030"});
|
|
drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_LEFT_EYE, {color: "#30FF30"});
|
|
drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_LEFT_EYEBROW, {color: "#30FF30"});
|
|
drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_FACE_OVAL, {color: "#E0E0E0"});
|
|
drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_LIPS, {color: "#E0E0E0"});
|
|
drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_RIGHT_IRIS, {color: "#FF3030"});
|
|
drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_LEFT_IRIS, {color: "#30FF30"});
|
|
}
|
|
}
|
|
}
|
|
drawBlendShapes(videoBlendShapes, results.faceBlendshapes);
|
|
// Call this function again to keep predicting when the browser is ready.
|
|
if (webcamRunning === true) {
|
|
window.requestAnimationFrame(predictWebcam);
|
|
}
|
|
}
|
|
|
|
function drawBlendShapes(el, blendShapes) {
|
|
if (!blendShapes.length) {
|
|
return;
|
|
}
|
|
|
|
let mappedJson = blendShapes[0].categories.reduce((obj, item) => {
|
|
obj[item.categoryName] = item.score;
|
|
return obj;
|
|
}, {});
|
|
|
|
mappedJson = applyLandmarksMapping(mappedJson)
|
|
|
|
|
|
var inputElements = document.getElementsByClassName('switchmove');
|
|
if (inputElements[0].checked) {
|
|
var send = {
|
|
"channel": channel,
|
|
"type": "trackingmotion",
|
|
"data": mappedJson,
|
|
"yaw": yawDegrees,
|
|
"pitch": pitchDegrees,
|
|
"roll": rollDegrees
|
|
}
|
|
} else {
|
|
var send = {
|
|
"channel": channel,
|
|
"type": "tracking",
|
|
"data": mappedJson,
|
|
"yaw": yawDegrees,
|
|
"pitch": pitchDegrees,
|
|
"roll": rollDegrees
|
|
}
|
|
}
|
|
|
|
EventBridge.emitWebEvent(JSON.stringify(send));
|
|
}
|
|
|
|
</script>
|
|
|
|
|
|
</body>
|
|
|
|
</html>
|