ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

mobile demo optimization and iris gestures

pull/134/head
Vladimir Mandic 2021-04-18 19:33:40 -04:00
parent 29451107d3
commit 203178e28f
24 changed files with 160 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGELOG.md
View File

@ -1,6 +1,6 @@
# @vladmandic/human # @vladmandic/human
Version: **1.5.2** Version: **1.6.0**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>** Author: **Vladimir Mandic <mandic00@live.com>**
@ -11,9 +11,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/04/16 mandic00@live.com ### **HEAD -> main** 2021/04/16 mandic00@live.com
- full rebuild
### **origin/main** 2021/04/16 mandic00@live.com - new look
- added benchmarks
- added node-multiprocess demo - added node-multiprocess demo
- fix image orientation - fix image orientation
- flat app style - flat app style

16
demo/helpers/menu.js
View File

@ -118,7 +118,7 @@ class Menu {
this.menu.appendChild(this.container); this.menu.appendChild(this.container);
if (typeof parent === 'object') parent.appendChild(this.menu); if (typeof parent === 'object') parent.appendChild(this.menu);
else document.getElementById(parent)?.appendChild(this.menu); else document.getElementById(parent).appendChild(this.menu);
} }
get newID() { get newID() {
@ -131,11 +131,11 @@ class Menu {
} }
get width() { get width() {
return this.menu?.offsetWidth || 0; return this.menu.offsetWidth || 0;
} }
get height() { get height() {
return this.menu?.offsetHeight || 0; return this.menu.offsetHeight || 0;
} }
hide() { hide() {
@ -203,8 +203,8 @@ class Menu {
el.innerHTML = `<div class="menu-checkbox"><input class="menu-checkbox" type="checkbox" id="${this.newID}" ${object[variable] ? 'checked' : ''}/><label class="menu-checkbox-label" for="${this.ID}"></label></div>${title}`; el.innerHTML = `<div class="menu-checkbox"><input class="menu-checkbox" type="checkbox" id="${this.newID}" ${object[variable] ? 'checked' : ''}/><label class="menu-checkbox-label" for="${this.ID}"></label></div>${title}`;
if (this.container) this.container.appendChild(el); if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => { el.addEventListener('change', (evt) => {
object[variable] = evt.target?.checked; object[variable] = evt.target.checked;
if (callback) callback(evt.target?.checked); if (callback) callback(evt.target.checked);
}); });
return el; return el;
} }
@ -223,7 +223,7 @@ class Menu {
el.style.fontVariant = document.body.style.fontVariant; el.style.fontVariant = document.body.style.fontVariant;
if (this.container) this.container.appendChild(el); if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => { el.addEventListener('change', (evt) => {
if (callback) callback(items[evt.target?.selectedIndex]); if (callback) callback(items[evt.target.selectedIndex]);
}); });
return el; return el;
} }
@ -235,9 +235,9 @@ class Menu {
if (this.container) this.container.appendChild(el); if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => { el.addEventListener('change', (evt) => {
if (evt.target) { if (evt.target) {
object[variable] = parseInt(evt.target?.value) === parseFloat(evt.target?.value) ? parseInt(evt.target?.value) : parseFloat(evt.target?.value); object[variable] = parseInt(evt.target.value) === parseFloat(evt.target.value) ? parseInt(evt.target.value) : parseFloat(evt.target.value);
evt.target.setAttribute('value', evt.target.value); evt.target.setAttribute('value', evt.target.value);
if (callback) callback(evt.target?.value); if (callback) callback(evt.target.value);
} }
}); });
el.input = el.children[0]; el.input = el.children[0];

25
demo/helpers/webrtc.js
View File

@ -17,18 +17,21 @@ async function webRTC(server, streamName, elementName) {
const connection = new RTCPeerConnection(); const connection = new RTCPeerConnection();
connection.oniceconnectionstatechange = () => log('connection', connection.iceConnectionState); connection.oniceconnectionstatechange = () => log('connection', connection.iceConnectionState);
connection.onnegotiationneeded = async () => { connection.onnegotiationneeded = async () => {
const offer = await connection.createOffer(); let offer;
await connection.setLocalDescription(offer); if (connection.localDescription) {
const res = await fetch(`${server}/stream/receiver/${suuid}`, { offer = await connection.createOffer();
method: 'POST', await connection.setLocalDescription(offer);
headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' }, const res = await fetch(`${server}/stream/receiver/${suuid}`, {
body: new URLSearchParams({ method: 'POST',
suuid: `${suuid}`, headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' },
data: `${btoa(connection.localDescription?.sdp || '')}`, body: new URLSearchParams({
}), suuid: `${suuid}`,
}); data: `${btoa(connection.localDescription.sdp || '')}`,
}),
});
}
const data = res && res.ok ? await res.text() : ''; const data = res && res.ok ? await res.text() : '';
if (data.length === 0) { if (data.length === 0 || !offer) {
log('cannot connect:', server); log('cannot connect:', server);
} else { } else {
connection.setRemoteDescription(new RTCSessionDescription({ connection.setRemoteDescription(new RTCSessionDescription({

4
demo/index.html
View File

@ -16,7 +16,7 @@
<style> <style>
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../assets/lato-light.woff2') } @font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../assets/lato-light.woff2') }
html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; } html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
body { margin: 0; background: black; color: white; overflow-x: hidden } body { margin: 0; background: black; color: white; overflow-x: hidden; width: 100vw; height: 100vh; }
body::-webkit-scrollbar { display: none; } body::-webkit-scrollbar { display: none; }
hr { width: 100%; } hr { width: 100%; }
.play { position: absolute; width: 256px; height: 256px; z-index: 9; bottom: 15%; left: 50%; margin-left: -125px; display: none; filter: grayscale(1); } .play { position: absolute; width: 256px; height: 256px; z-index: 9; bottom: 15%; left: 50%; margin-left: -125px; display: none; filter: grayscale(1); }
@ -28,7 +28,7 @@
.status { position: absolute; width: 100vw; bottom: 10%; text-align: center; font-size: 4rem; font-weight: 100; text-shadow: 2px 2px #303030; } .status { position: absolute; width: 100vw; bottom: 10%; text-align: center; font-size: 4rem; font-weight: 100; text-shadow: 2px 2px #303030; }
.thumbnail { margin: 8px; box-shadow: 0 0 4px 4px dimgrey; } .thumbnail { margin: 8px; box-shadow: 0 0 4px 4px dimgrey; }
.thumbnail:hover { box-shadow: 0 0 8px 8px dimgrey; filter: grayscale(1); } .thumbnail:hover { box-shadow: 0 0 8px 8px dimgrey; filter: grayscale(1); }
.log { position: absolute; bottom: 0; margin: 0.4rem; font-size: 0.9rem; } .log { position: absolute; bottom: 0; margin: 0.4rem 0.4rem 0 0.4rem; font-size: 0.9rem; }
.menubar { width: 100vw; background: #303030; display: flex; justify-content: space-evenly; text-align: center; padding: 8px; cursor: pointer; } .menubar { width: 100vw; background: #303030; display: flex; justify-content: space-evenly; text-align: center; padding: 8px; cursor: pointer; }
.samples-container { display: flex; flex-wrap: wrap; } .samples-container { display: flex; flex-wrap: wrap; }
.video { display: none; } .video { display: none; }

42
demo/index.js
View File

@ -101,7 +101,8 @@ const compare = { enabled: false, original: null };
async function calcSimmilariry(result) { async function calcSimmilariry(result) {
document.getElementById('compare-container').style.display = compare.enabled ? 'block' : 'none'; document.getElementById('compare-container').style.display = compare.enabled ? 'block' : 'none';
if (!compare.enabled) return; if (!compare.enabled) return;
if (!(result?.face?.length > 0) || (result?.face[0]?.embedding?.length <= 64)) return; if (!result || !result.face || result.face[0].embedding) return;
if (!(result.face.length > 0) || (result.face[0].embedding.length <= 64)) return;
if (!compare.original) { if (!compare.original) {
compare.original = result; compare.original = result;
log('setting face compare baseline:', result.face[0]); log('setting face compare baseline:', result.face[0]);
@ -120,7 +121,7 @@ async function calcSimmilariry(result) {
document.getElementById('compare-canvas').getContext('2d').drawImage(compare.original.canvas, 0, 0, 200, 200); document.getElementById('compare-canvas').getContext('2d').drawImage(compare.original.canvas, 0, 0, 200, 200);
} }
} }
const similarity = human.similarity(compare.original?.face[0]?.embedding, result?.face[0]?.embedding); const similarity = human.similarity(compare.original.face[0].embedding, result.face[0].embedding);
document.getElementById('similarity').innerText = `similarity: ${Math.trunc(1000 * similarity) / 10}%`; document.getElementById('similarity').innerText = `similarity: ${Math.trunc(1000 * similarity) / 10}%`;
} }
@ -250,7 +251,7 @@ async function setupCamera() {
const track = stream.getVideoTracks()[0]; const track = stream.getVideoTracks()[0];
const settings = track.getSettings(); const settings = track.getSettings();
// log('camera constraints:', constraints, 'window:', { width: window.innerWidth, height: window.innerHeight }, 'settings:', settings, 'track:', track); // log('camera constraints:', constraints, 'window:', { width: window.innerWidth, height: window.innerHeight }, 'settings:', settings, 'track:', track);
ui.camera = { name: track.label?.toLowerCase(), width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' }; ui.camera = { name: track.label.toLowerCase(), width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
return new Promise((resolve) => { return new Promise((resolve) => {
video.onloadeddata = async () => { video.onloadeddata = async () => {
video.width = video.videoWidth; video.width = video.videoWidth;
@ -327,7 +328,7 @@ function runHumanDetect(input, canvas, timestamp) {
// if we want to continue and camera not ready, retry in 0.5sec, else just give up // if we want to continue and camera not ready, retry in 0.5sec, else just give up
if (input.paused) log('camera paused'); if (input.paused) log('camera paused');
else if ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500); else if ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500);
else log(`camera not ready: track state: ${input.srcObject?.getVideoTracks()[0].readyState} stream state: ${input.readyState}`); else log(`camera not ready: track state: ${input.srcObject.getVideoTracks()[0].readyState} stream state: ${input.readyState}`);
clearTimeout(ui.drawThread); clearTimeout(ui.drawThread);
ui.drawThread = null; ui.drawThread = null;
log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw); log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw);
@ -442,11 +443,12 @@ async function detectSampleImages() {
function setupMenu() { function setupMenu() {
const x = [`${document.getElementById('btnDisplay').offsetLeft}px`, `${document.getElementById('btnImage').offsetLeft}px`, `${document.getElementById('btnProcess').offsetLeft}px`, `${document.getElementById('btnModel').offsetLeft}px`]; const x = [`${document.getElementById('btnDisplay').offsetLeft}px`, `${document.getElementById('btnImage').offsetLeft}px`, `${document.getElementById('btnProcess').offsetLeft}px`, `${document.getElementById('btnModel').offsetLeft}px`];
const top = `${document.getElementById('menubar').offsetHeight - 3}px`;
const top = `${document.getElementById('menubar').clientHeight}px`;
menu.display = new Menu(document.body, '', { top, left: x[0] }); menu.display = new Menu(document.body, '', { top, left: x[0] });
menu.display.addBool('perf monitor', ui, 'bench', (val) => ui.bench = val); menu.display.addBool('perf monitor', ui, 'bench', (val) => ui.bench = val);
menu.display.addBool('buffered output', ui, 'buffered', (val) => ui.buffered = val); menu.display.addBool('buffer output', ui, 'buffered', (val) => ui.buffered = val);
menu.display.addBool('crop & scale', ui, 'crop', (val) => { menu.display.addBool('crop & scale', ui, 'crop', (val) => {
ui.crop = val; ui.crop = val;
setupCamera(); setupCamera();
@ -456,8 +458,8 @@ function setupMenu() {
setupCamera(); setupCamera();
}); });
menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">'); menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.display.addBool('use 3D depth', human.draw.options, 'useDepth'); menu.display.addBool('use depth', human.draw.options, 'useDepth');
menu.display.addBool('draw with curves', human.draw.options, 'useCurves'); menu.display.addBool('use curves', human.draw.options, 'useCurves');
menu.display.addBool('print labels', human.draw.options, 'drawLabels'); menu.display.addBool('print labels', human.draw.options, 'drawLabels');
menu.display.addBool('draw points', human.draw.options, 'drawPoints'); menu.display.addBool('draw points', human.draw.options, 'drawPoints');
menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes'); menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes');
@ -557,17 +559,28 @@ function setupMenu() {
} }
async function resize() { async function resize() {
window.onresize = null;
const viewportScale = Math.min(1, Math.round(100 * window.innerWidth / 960) / 100);
const viewport = document.querySelector('meta[name=viewport]');
viewport.setAttribute('content', `width=device-width, shrink-to-fit=yes, minimum-scale=0.2, maximum-scale=2.0, user-scalable=yes, initial-scale=${viewportScale}`);
const x = [`${document.getElementById('btnDisplay').offsetLeft}px`, `${document.getElementById('btnImage').offsetLeft}px`, `${document.getElementById('btnProcess').offsetLeft}px`, `${document.getElementById('btnModel').offsetLeft}px`]; const x = [`${document.getElementById('btnDisplay').offsetLeft}px`, `${document.getElementById('btnImage').offsetLeft}px`, `${document.getElementById('btnProcess').offsetLeft}px`, `${document.getElementById('btnModel').offsetLeft}px`];
const top = `${document.getElementById('menubar').clientHeight - 3}px`;
menu.display.menu.style.top = top;
menu.image.menu.style.top = top;
menu.process.menu.style.top = top;
menu.models.menu.style.top = top;
menu.display.menu.style.left = x[0]; menu.display.menu.style.left = x[0];
menu.image.menu.style.left = x[1]; menu.image.menu.style.left = x[1];
menu.process.menu.style.left = x[2]; menu.process.menu.style.left = x[2];
menu.models.menu.style.left = x[3]; menu.models.menu.style.left = x[3];
const viewportScale = Math.min(1, Math.round(100 * window.innerWidth / 960) / 100);
const viewport = document.querySelector('meta[name=viewport]'); const fontSize = Math.trunc(10 * (1 - viewportScale)) + 16;
viewport.setAttribute('content', `width=device-width, shrink-to-fit=yes, minimum-scale=0.2, maximum-scale=2.0, user-scalable=yes, initial-scale=${viewportScale}`); document.documentElement.style.fontSize = `${fontSize}px`;
// console.log('view', viewportScale, window.innerWidth, viewport);
// document.body.style.MozTransform = `scale(${viewportScale})`; human.draw.options.font = `small-caps ${fontSize + 4}px "Segoe UI"`;
// document.body.style.zoom = `scale(${viewportScale})`;
setupCamera(); setupCamera();
} }
@ -638,6 +651,7 @@ async function main() {
document.getElementById('loader').style.display = 'none'; document.getElementById('loader').style.display = 'none';
document.getElementById('play').style.display = 'block'; document.getElementById('play').style.display = 'block';
log('demo ready...'); log('demo ready...');
for (const m of Object.values(menu)) m.hide();
} }
window.onload = main; window.onload = main;

6
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

14
dist/human.node-gpu.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node-gpu.js.map vendored
View File

File diff suppressed because one or more lines are too long

14
dist/human.node-wasm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node-wasm.js.map vendored
View File

File diff suppressed because one or more lines are too long

14
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

4
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@vladmandic/human", "name": "@vladmandic/human",
"version": "1.5.2", "version": "1.6.0",
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition", "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
"sideEffects": false, "sideEffects": false,
"main": "dist/human.node.js", "main": "dist/human.node.js",
@ -68,7 +68,7 @@
"canvas": "^2.7.0", "canvas": "^2.7.0",
"chokidar": "^3.5.1", "chokidar": "^3.5.1",
"dayjs": "^1.10.4", "dayjs": "^1.10.4",
"esbuild": "^0.11.11", "esbuild": "^0.11.12",
"eslint": "^7.24.0", "eslint": "^7.24.0",
"eslint-config-airbnb-base": "^14.2.1", "eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.22.1", "eslint-plugin-import": "^2.22.1",

17
server/build.log
View File

@ -49,3 +49,20 @@
2021-04-16 18:02:37 INFO:  Generate types: ["src/human.ts"] 2021-04-16 18:02:37 INFO:  Generate types: ["src/human.ts"]
2021-04-16 18:02:43 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"] 2021-04-16 18:02:43 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-16 18:02:43 INFO:  Generate TypeDocs: ["src/human.ts"] 2021-04-16 18:02:43 INFO:  Generate TypeDocs: ["src/human.ts"]
2021-04-18 19:33:07 INFO:  @vladmandic/human version 1.6.0
2021-04-18 19:33:07 INFO:  User: vlado Platform: linux Arch: x64 Node: v15.7.0
2021-04-18 19:33:07 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true,"sourcemap":true,"bundle":true,"metafile":true,"target":"es2018"}
2021-04-18 19:33:07 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":733,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:07 STATE: Build for: node type: node: {"imports":46,"importBytes":546322,"outputBytes":304931,"outputFiles":"dist/human.node.js"}
2021-04-18 19:33:07 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":737,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:07 STATE: Build for: nodeGPU type: node: {"imports":46,"importBytes":546326,"outputBytes":304939,"outputFiles":"dist/human.node-gpu.js"}
2021-04-18 19:33:07 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":783,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:07 STATE: Build for: nodeWASM type: node: {"imports":46,"importBytes":546372,"outputBytes":304983,"outputFiles":"dist/human.node-wasm.js"}
2021-04-18 19:33:07 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:07 STATE: Build for: browserNoBundle type: esm: {"imports":46,"importBytes":546983,"outputBytes":304929,"outputFiles":"dist/human.esm-nobundle.js"}
2021-04-18 19:33:08 STATE: Build for: browserBundle type: tfjs: {"modules":1262,"moduleBytes":4068263,"imports":7,"importBytes":2488,"outputBytes":1097287,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:08 STATE: Build for: browserBundle type: iife: {"imports":46,"importBytes":1642876,"outputBytes":1398352,"outputFiles":"dist/human.js"}
2021-04-18 19:33:09 STATE: Build for: browserBundle type: esm: {"imports":46,"importBytes":1642876,"outputBytes":1398310,"outputFiles":"dist/human.esm.js"}
2021-04-18 19:33:09 INFO:  Generate types: ["src/human.ts"]
2021-04-18 19:33:14 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-18 19:33:14 INFO:  Generate TypeDocs: ["src/human.ts"]

26
src/draw/draw.ts
View File

@ -41,6 +41,7 @@ export interface DrawOptions {
useCurves: Boolean, useCurves: Boolean,
bufferedOutput: Boolean, bufferedOutput: Boolean,
useRawBoxes: Boolean, useRawBoxes: Boolean,
calculateHandBox: Boolean,
} }
export const options: DrawOptions = { export const options: DrawOptions = {
@ -61,6 +62,7 @@ export const options: DrawOptions = {
useCurves: <Boolean>false, useCurves: <Boolean>false,
bufferedOutput: <Boolean>false, bufferedOutput: <Boolean>false,
useRawBoxes: <Boolean>false, useRawBoxes: <Boolean>false,
calculateHandBox: <Boolean>true,
}; };
function point(ctx, x, y, z = 0, localOptions) { function point(ctx, x, y, z = 0, localOptions) {
@ -359,15 +361,31 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<any>, draw
if (localOptions.drawBoxes) { if (localOptions.drawBoxes) {
ctx.strokeStyle = localOptions.color; ctx.strokeStyle = localOptions.color;
ctx.fillStyle = localOptions.color; ctx.fillStyle = localOptions.color;
if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3], localOptions); let box;
else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions); if (!localOptions.calculateHandBox) {
box = localOptions.useRawBoxes ? h.boxRaw : h.box;
} else {
box = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0];
if (h.landmarks && h.landmarks.length > 0) {
for (const pt of h.landmarks) {
if (pt[0] < box[0]) box[0] = pt[0];
if (pt[1] < box[1]) box[1] = pt[1];
if (pt[0] > box[2]) box[2] = pt[0];
if (pt[1] > box[3]) box[3] = pt[1];
}
box[2] -= box[0];
box[3] -= box[1];
}
}
if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * box[0], inCanvas.height * box[1], inCanvas.width * box[2], inCanvas.height * box[3], localOptions);
else rect(ctx, box[0], box[1], box[2], box[3], localOptions);
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
if (localOptions.shadowColor && localOptions.shadowColor !== '') { if (localOptions.shadowColor && localOptions.shadowColor !== '') {
ctx.fillStyle = localOptions.shadowColor; ctx.fillStyle = localOptions.shadowColor;
ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]); ctx.fillText('hand', box[0] + 3, 1 + box[1] + localOptions.lineHeight, box[2]);
} }
ctx.fillStyle = localOptions.labelColor; ctx.fillStyle = localOptions.labelColor;
ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]); ctx.fillText('hand', box[0] + 2, 0 + box[1] + localOptions.lineHeight, box[2]);
} }
ctx.stroke(); ctx.stroke();
} }

10
src/gesture/gesture.ts
View File

@ -24,7 +24,7 @@ export const face = (res) => {
for (let i = 0; i < res.length; i++) { for (let i = 0; i < res.length; i++) {
if (res[i].mesh && res[i].mesh.length > 0) { if (res[i].mesh && res[i].mesh.length > 0) {
const eyeFacing = res[i].mesh[33][2] - res[i].mesh[263][2]; const eyeFacing = res[i].mesh[33][2] - res[i].mesh[263][2];
if (Math.abs(eyeFacing) < 10) gestures.push({ face: i, gesture: 'facing camera' }); if (Math.abs(eyeFacing) < 10) gestures.push({ face: i, gesture: 'facing center' });
else gestures.push({ face: i, gesture: `facing ${eyeFacing < 0 ? 'right' : 'left'}` }); else gestures.push({ face: i, gesture: `facing ${eyeFacing < 0 ? 'right' : 'left'}` });
const openLeft = Math.abs(res[i].mesh[374][1] - res[i].mesh[386][1]) / Math.abs(res[i].mesh[443][1] - res[i].mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord const openLeft = Math.abs(res[i].mesh[374][1] - res[i].mesh[386][1]) / Math.abs(res[i].mesh[443][1] - res[i].mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' }); if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' });
@ -53,7 +53,13 @@ export const iris = (res) => {
const areaRight = Math.abs(sizeXRight * sizeYRight); const areaRight = Math.abs(sizeXRight * sizeYRight);
const difference = Math.abs(areaLeft - areaRight) / Math.max(areaLeft, areaRight); const difference = Math.abs(areaLeft - areaRight) / Math.max(areaLeft, areaRight);
if (difference < 0.25) gestures.push({ iris: i, gesture: 'looking at camera' }); if (difference < 0.25) gestures.push({ iris: i, gesture: 'facing center' });
const rightIrisCenterX = Math.abs(res[i].mesh[33][0] - res[i].annotations.rightEyeIris[0][0]) / res[i].annotations.rightEyeIris[0][0];
const leftIrisCenterX = Math.abs(res[i].mesh[263][0] - res[i].annotations.leftEyeIris[0][0]) / res[i].annotations.leftEyeIris[0][0];
if (leftIrisCenterX > 0.025 && rightIrisCenterX > 0.025) gestures.push({ iris: i, gesture: 'looking center' });
else if (leftIrisCenterX > 0.025) gestures.push({ iris: i, gesture: 'looking right' });
else if (rightIrisCenterX > 0.025) gestures.push({ iris: i, gesture: 'looking left' });
} }
return gestures; return gestures;
}; };

2
typedoc/assets/js/search.js
View File

File diff suppressed because one or more lines are too long

11
typedoc/interfaces/drawoptions.html
View File

@ -104,6 +104,7 @@
<h3>Properties</h3> <h3>Properties</h3>
<ul class="tsd-index-list"> <ul class="tsd-index-list">
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#bufferedoutput" class="tsd-kind-icon">buffered<wbr>Output</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#bufferedoutput" class="tsd-kind-icon">buffered<wbr>Output</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#calculatehandbox" class="tsd-kind-icon">calculate<wbr>Hand<wbr>Box</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#color" class="tsd-kind-icon">color</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#color" class="tsd-kind-icon">color</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawboxes" class="tsd-kind-icon">draw<wbr>Boxes</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawboxes" class="tsd-kind-icon">draw<wbr>Boxes</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawlabels" class="tsd-kind-icon">draw<wbr>Labels</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawlabels" class="tsd-kind-icon">draw<wbr>Labels</a></li>
@ -134,6 +135,13 @@
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
</section> </section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="calculatehandbox" class="tsd-anchor"></a>
<h3>calculate<wbr>Hand<wbr>Box</h3>
<div class="tsd-signature tsd-kind-icon">calculate<wbr>Hand<wbr>Box<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Boolean</span></div>
<aside class="tsd-sources">
</aside>
</section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="color" class="tsd-anchor"></a> <a name="color" class="tsd-anchor"></a>
<h3>color</h3> <h3>color</h3>
@ -275,6 +283,9 @@
<li class=" tsd-kind-property tsd-parent-kind-interface"> <li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="drawoptions.html#bufferedoutput" class="tsd-kind-icon">buffered<wbr>Output</a> <a href="drawoptions.html#bufferedoutput" class="tsd-kind-icon">buffered<wbr>Output</a>
</li> </li>
<li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="drawoptions.html#calculatehandbox" class="tsd-kind-icon">calculate<wbr>Hand<wbr>Box</a>
</li>
<li class=" tsd-kind-property tsd-parent-kind-interface"> <li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="drawoptions.html#color" class="tsd-kind-icon">color</a> <a href="drawoptions.html#color" class="tsd-kind-icon">color</a>
</li> </li>

1
types/draw/draw.d.ts vendored
View File

@ -37,6 +37,7 @@ export interface DrawOptions {
useCurves: Boolean; useCurves: Boolean;
bufferedOutput: Boolean; bufferedOutput: Boolean;
useRawBoxes: Boolean; useRawBoxes: Boolean;
calculateHandBox: Boolean;
} }
export declare const options: DrawOptions; export declare const options: DrawOptions;
export declare function gesture(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions): Promise<void>; export declare function gesture(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions): Promise<void>;