mobile demo optimization and iris gestures

pull/134/head
Vladimir Mandic 2021-04-18 19:33:40 -04:00
parent 29451107d3
commit 203178e28f
24 changed files with 160 additions and 90 deletions

View File

@ -1,6 +1,6 @@
# @vladmandic/human
Version: **1.5.2**
Version: **1.6.0**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>**
@ -11,9 +11,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/04/16 mandic00@live.com
### **origin/main** 2021/04/16 mandic00@live.com
- full rebuild
- new look
- added benchmarks
- added node-multiprocess demo
- fix image orientation
- flat app style

View File

@ -118,7 +118,7 @@ class Menu {
this.menu.appendChild(this.container);
if (typeof parent === 'object') parent.appendChild(this.menu);
else document.getElementById(parent)?.appendChild(this.menu);
else document.getElementById(parent).appendChild(this.menu);
}
get newID() {
@ -131,11 +131,11 @@ class Menu {
}
get width() {
return this.menu?.offsetWidth || 0;
return this.menu.offsetWidth || 0;
}
get height() {
return this.menu?.offsetHeight || 0;
return this.menu.offsetHeight || 0;
}
hide() {
@ -203,8 +203,8 @@ class Menu {
el.innerHTML = `<div class="menu-checkbox"><input class="menu-checkbox" type="checkbox" id="${this.newID}" ${object[variable] ? 'checked' : ''}/><label class="menu-checkbox-label" for="${this.ID}"></label></div>${title}`;
if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => {
object[variable] = evt.target?.checked;
if (callback) callback(evt.target?.checked);
object[variable] = evt.target.checked;
if (callback) callback(evt.target.checked);
});
return el;
}
@ -223,7 +223,7 @@ class Menu {
el.style.fontVariant = document.body.style.fontVariant;
if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => {
if (callback) callback(items[evt.target?.selectedIndex]);
if (callback) callback(items[evt.target.selectedIndex]);
});
return el;
}
@ -235,9 +235,9 @@ class Menu {
if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => {
if (evt.target) {
object[variable] = parseInt(evt.target?.value) === parseFloat(evt.target?.value) ? parseInt(evt.target?.value) : parseFloat(evt.target?.value);
object[variable] = parseInt(evt.target.value) === parseFloat(evt.target.value) ? parseInt(evt.target.value) : parseFloat(evt.target.value);
evt.target.setAttribute('value', evt.target.value);
if (callback) callback(evt.target?.value);
if (callback) callback(evt.target.value);
}
});
el.input = el.children[0];

View File

@ -17,18 +17,21 @@ async function webRTC(server, streamName, elementName) {
const connection = new RTCPeerConnection();
connection.oniceconnectionstatechange = () => log('connection', connection.iceConnectionState);
connection.onnegotiationneeded = async () => {
const offer = await connection.createOffer();
await connection.setLocalDescription(offer);
const res = await fetch(`${server}/stream/receiver/${suuid}`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' },
body: new URLSearchParams({
suuid: `${suuid}`,
data: `${btoa(connection.localDescription?.sdp || '')}`,
}),
});
let offer;
if (connection.localDescription) {
offer = await connection.createOffer();
await connection.setLocalDescription(offer);
const res = await fetch(`${server}/stream/receiver/${suuid}`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' },
body: new URLSearchParams({
suuid: `${suuid}`,
data: `${btoa(connection.localDescription.sdp || '')}`,
}),
});
}
const data = res && res.ok ? await res.text() : '';
if (data.length === 0) {
if (data.length === 0 || !offer) {
log('cannot connect:', server);
} else {
connection.setRemoteDescription(new RTCSessionDescription({

View File

@ -16,7 +16,7 @@
<style>
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../assets/lato-light.woff2') }
html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
body { margin: 0; background: black; color: white; overflow-x: hidden }
body { margin: 0; background: black; color: white; overflow-x: hidden; width: 100vw; height: 100vh; }
body::-webkit-scrollbar { display: none; }
hr { width: 100%; }
.play { position: absolute; width: 256px; height: 256px; z-index: 9; bottom: 15%; left: 50%; margin-left: -125px; display: none; filter: grayscale(1); }
@ -28,7 +28,7 @@
.status { position: absolute; width: 100vw; bottom: 10%; text-align: center; font-size: 4rem; font-weight: 100; text-shadow: 2px 2px #303030; }
.thumbnail { margin: 8px; box-shadow: 0 0 4px 4px dimgrey; }
.thumbnail:hover { box-shadow: 0 0 8px 8px dimgrey; filter: grayscale(1); }
.log { position: absolute; bottom: 0; margin: 0.4rem; font-size: 0.9rem; }
.log { position: absolute; bottom: 0; margin: 0.4rem 0.4rem 0 0.4rem; font-size: 0.9rem; }
.menubar { width: 100vw; background: #303030; display: flex; justify-content: space-evenly; text-align: center; padding: 8px; cursor: pointer; }
.samples-container { display: flex; flex-wrap: wrap; }
.video { display: none; }

View File

@ -101,7 +101,8 @@ const compare = { enabled: false, original: null };
async function calcSimmilariry(result) {
document.getElementById('compare-container').style.display = compare.enabled ? 'block' : 'none';
if (!compare.enabled) return;
if (!(result?.face?.length > 0) || (result?.face[0]?.embedding?.length <= 64)) return;
if (!result || !result.face || result.face[0].embedding) return;
if (!(result.face.length > 0) || (result.face[0].embedding.length <= 64)) return;
if (!compare.original) {
compare.original = result;
log('setting face compare baseline:', result.face[0]);
@ -120,7 +121,7 @@ async function calcSimmilariry(result) {
document.getElementById('compare-canvas').getContext('2d').drawImage(compare.original.canvas, 0, 0, 200, 200);
}
}
const similarity = human.similarity(compare.original?.face[0]?.embedding, result?.face[0]?.embedding);
const similarity = human.similarity(compare.original.face[0].embedding, result.face[0].embedding);
document.getElementById('similarity').innerText = `similarity: ${Math.trunc(1000 * similarity) / 10}%`;
}
@ -250,7 +251,7 @@ async function setupCamera() {
const track = stream.getVideoTracks()[0];
const settings = track.getSettings();
// log('camera constraints:', constraints, 'window:', { width: window.innerWidth, height: window.innerHeight }, 'settings:', settings, 'track:', track);
ui.camera = { name: track.label?.toLowerCase(), width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
ui.camera = { name: track.label.toLowerCase(), width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
return new Promise((resolve) => {
video.onloadeddata = async () => {
video.width = video.videoWidth;
@ -327,7 +328,7 @@ function runHumanDetect(input, canvas, timestamp) {
// if we want to continue and camera not ready, retry in 0.5sec, else just give up
if (input.paused) log('camera paused');
else if ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500);
else log(`camera not ready: track state: ${input.srcObject?.getVideoTracks()[0].readyState} stream state: ${input.readyState}`);
else log(`camera not ready: track state: ${input.srcObject.getVideoTracks()[0].readyState} stream state: ${input.readyState}`);
clearTimeout(ui.drawThread);
ui.drawThread = null;
log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw);
@ -442,11 +443,12 @@ async function detectSampleImages() {
function setupMenu() {
const x = [`${document.getElementById('btnDisplay').offsetLeft}px`, `${document.getElementById('btnImage').offsetLeft}px`, `${document.getElementById('btnProcess').offsetLeft}px`, `${document.getElementById('btnModel').offsetLeft}px`];
const top = `${document.getElementById('menubar').offsetHeight - 3}px`;
const top = `${document.getElementById('menubar').clientHeight}px`;
menu.display = new Menu(document.body, '', { top, left: x[0] });
menu.display.addBool('perf monitor', ui, 'bench', (val) => ui.bench = val);
menu.display.addBool('buffered output', ui, 'buffered', (val) => ui.buffered = val);
menu.display.addBool('buffer output', ui, 'buffered', (val) => ui.buffered = val);
menu.display.addBool('crop & scale', ui, 'crop', (val) => {
ui.crop = val;
setupCamera();
@ -456,8 +458,8 @@ function setupMenu() {
setupCamera();
});
menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.display.addBool('use 3D depth', human.draw.options, 'useDepth');
menu.display.addBool('draw with curves', human.draw.options, 'useCurves');
menu.display.addBool('use depth', human.draw.options, 'useDepth');
menu.display.addBool('use curves', human.draw.options, 'useCurves');
menu.display.addBool('print labels', human.draw.options, 'drawLabels');
menu.display.addBool('draw points', human.draw.options, 'drawPoints');
menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes');
@ -557,17 +559,28 @@ function setupMenu() {
}
async function resize() {
window.onresize = null;
const viewportScale = Math.min(1, Math.round(100 * window.innerWidth / 960) / 100);
const viewport = document.querySelector('meta[name=viewport]');
viewport.setAttribute('content', `width=device-width, shrink-to-fit=yes, minimum-scale=0.2, maximum-scale=2.0, user-scalable=yes, initial-scale=${viewportScale}`);
const x = [`${document.getElementById('btnDisplay').offsetLeft}px`, `${document.getElementById('btnImage').offsetLeft}px`, `${document.getElementById('btnProcess').offsetLeft}px`, `${document.getElementById('btnModel').offsetLeft}px`];
const top = `${document.getElementById('menubar').clientHeight - 3}px`;
menu.display.menu.style.top = top;
menu.image.menu.style.top = top;
menu.process.menu.style.top = top;
menu.models.menu.style.top = top;
menu.display.menu.style.left = x[0];
menu.image.menu.style.left = x[1];
menu.process.menu.style.left = x[2];
menu.models.menu.style.left = x[3];
const viewportScale = Math.min(1, Math.round(100 * window.innerWidth / 960) / 100);
const viewport = document.querySelector('meta[name=viewport]');
viewport.setAttribute('content', `width=device-width, shrink-to-fit=yes, minimum-scale=0.2, maximum-scale=2.0, user-scalable=yes, initial-scale=${viewportScale}`);
// console.log('view', viewportScale, window.innerWidth, viewport);
// document.body.style.MozTransform = `scale(${viewportScale})`;
// document.body.style.zoom = `scale(${viewportScale})`;
const fontSize = Math.trunc(10 * (1 - viewportScale)) + 16;
document.documentElement.style.fontSize = `${fontSize}px`;
human.draw.options.font = `small-caps ${fontSize + 4}px "Segoe UI"`;
setupCamera();
}
@ -638,6 +651,7 @@ async function main() {
document.getElementById('loader').style.display = 'none';
document.getElementById('play').style.display = 'block';
log('demo ready...');
for (const m of Object.values(menu)) m.hide();
}
window.onload = main;

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

6
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

6
dist/human.js vendored

File diff suppressed because one or more lines are too long

4
dist/human.js.map vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

14
dist/human.node.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -1,6 +1,6 @@
{
"name": "@vladmandic/human",
"version": "1.5.2",
"version": "1.6.0",
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
"sideEffects": false,
"main": "dist/human.node.js",
@ -68,7 +68,7 @@
"canvas": "^2.7.0",
"chokidar": "^3.5.1",
"dayjs": "^1.10.4",
"esbuild": "^0.11.11",
"esbuild": "^0.11.12",
"eslint": "^7.24.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.22.1",

View File

@ -49,3 +49,20 @@
2021-04-16 18:02:37 INFO:  Generate types: ["src/human.ts"]
2021-04-16 18:02:43 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-16 18:02:43 INFO:  Generate TypeDocs: ["src/human.ts"]
2021-04-18 19:33:07 INFO:  @vladmandic/human version 1.6.0
2021-04-18 19:33:07 INFO:  User: vlado Platform: linux Arch: x64 Node: v15.7.0
2021-04-18 19:33:07 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true,"sourcemap":true,"bundle":true,"metafile":true,"target":"es2018"}
2021-04-18 19:33:07 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":733,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:07 STATE: Build for: node type: node: {"imports":46,"importBytes":546322,"outputBytes":304931,"outputFiles":"dist/human.node.js"}
2021-04-18 19:33:07 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":737,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:07 STATE: Build for: nodeGPU type: node: {"imports":46,"importBytes":546326,"outputBytes":304939,"outputFiles":"dist/human.node-gpu.js"}
2021-04-18 19:33:07 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":783,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:07 STATE: Build for: nodeWASM type: node: {"imports":46,"importBytes":546372,"outputBytes":304983,"outputFiles":"dist/human.node-wasm.js"}
2021-04-18 19:33:07 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:07 STATE: Build for: browserNoBundle type: esm: {"imports":46,"importBytes":546983,"outputBytes":304929,"outputFiles":"dist/human.esm-nobundle.js"}
2021-04-18 19:33:08 STATE: Build for: browserBundle type: tfjs: {"modules":1262,"moduleBytes":4068263,"imports":7,"importBytes":2488,"outputBytes":1097287,"outputFiles":"dist/tfjs.esm.js"}
2021-04-18 19:33:08 STATE: Build for: browserBundle type: iife: {"imports":46,"importBytes":1642876,"outputBytes":1398352,"outputFiles":"dist/human.js"}
2021-04-18 19:33:09 STATE: Build for: browserBundle type: esm: {"imports":46,"importBytes":1642876,"outputBytes":1398310,"outputFiles":"dist/human.esm.js"}
2021-04-18 19:33:09 INFO:  Generate types: ["src/human.ts"]
2021-04-18 19:33:14 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-18 19:33:14 INFO:  Generate TypeDocs: ["src/human.ts"]

View File

@ -41,6 +41,7 @@ export interface DrawOptions {
useCurves: Boolean,
bufferedOutput: Boolean,
useRawBoxes: Boolean,
calculateHandBox: Boolean,
}
export const options: DrawOptions = {
@ -61,6 +62,7 @@ export const options: DrawOptions = {
useCurves: <Boolean>false,
bufferedOutput: <Boolean>false,
useRawBoxes: <Boolean>false,
calculateHandBox: <Boolean>true,
};
function point(ctx, x, y, z = 0, localOptions) {
@ -359,15 +361,31 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<any>, draw
if (localOptions.drawBoxes) {
ctx.strokeStyle = localOptions.color;
ctx.fillStyle = localOptions.color;
if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3], localOptions);
else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions);
let box;
if (!localOptions.calculateHandBox) {
box = localOptions.useRawBoxes ? h.boxRaw : h.box;
} else {
box = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0];
if (h.landmarks && h.landmarks.length > 0) {
for (const pt of h.landmarks) {
if (pt[0] < box[0]) box[0] = pt[0];
if (pt[1] < box[1]) box[1] = pt[1];
if (pt[0] > box[2]) box[2] = pt[0];
if (pt[1] > box[3]) box[3] = pt[1];
}
box[2] -= box[0];
box[3] -= box[1];
}
}
if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * box[0], inCanvas.height * box[1], inCanvas.width * box[2], inCanvas.height * box[3], localOptions);
else rect(ctx, box[0], box[1], box[2], box[3], localOptions);
if (localOptions.drawLabels) {
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
ctx.fillStyle = localOptions.shadowColor;
ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]);
ctx.fillText('hand', box[0] + 3, 1 + box[1] + localOptions.lineHeight, box[2]);
}
ctx.fillStyle = localOptions.labelColor;
ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]);
ctx.fillText('hand', box[0] + 2, 0 + box[1] + localOptions.lineHeight, box[2]);
}
ctx.stroke();
}

View File

@ -24,7 +24,7 @@ export const face = (res) => {
for (let i = 0; i < res.length; i++) {
if (res[i].mesh && res[i].mesh.length > 0) {
const eyeFacing = res[i].mesh[33][2] - res[i].mesh[263][2];
if (Math.abs(eyeFacing) < 10) gestures.push({ face: i, gesture: 'facing camera' });
if (Math.abs(eyeFacing) < 10) gestures.push({ face: i, gesture: 'facing center' });
else gestures.push({ face: i, gesture: `facing ${eyeFacing < 0 ? 'right' : 'left'}` });
const openLeft = Math.abs(res[i].mesh[374][1] - res[i].mesh[386][1]) / Math.abs(res[i].mesh[443][1] - res[i].mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' });
@ -53,7 +53,13 @@ export const iris = (res) => {
const areaRight = Math.abs(sizeXRight * sizeYRight);
const difference = Math.abs(areaLeft - areaRight) / Math.max(areaLeft, areaRight);
if (difference < 0.25) gestures.push({ iris: i, gesture: 'looking at camera' });
if (difference < 0.25) gestures.push({ iris: i, gesture: 'facing center' });
const rightIrisCenterX = Math.abs(res[i].mesh[33][0] - res[i].annotations.rightEyeIris[0][0]) / res[i].annotations.rightEyeIris[0][0];
const leftIrisCenterX = Math.abs(res[i].mesh[263][0] - res[i].annotations.leftEyeIris[0][0]) / res[i].annotations.leftEyeIris[0][0];
if (leftIrisCenterX > 0.025 && rightIrisCenterX > 0.025) gestures.push({ iris: i, gesture: 'looking center' });
else if (leftIrisCenterX > 0.025) gestures.push({ iris: i, gesture: 'looking right' });
else if (rightIrisCenterX > 0.025) gestures.push({ iris: i, gesture: 'looking left' });
}
return gestures;
};

File diff suppressed because one or more lines are too long

View File

@ -104,6 +104,7 @@
<h3>Properties</h3>
<ul class="tsd-index-list">
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#bufferedoutput" class="tsd-kind-icon">buffered<wbr>Output</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#calculatehandbox" class="tsd-kind-icon">calculate<wbr>Hand<wbr>Box</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#color" class="tsd-kind-icon">color</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawboxes" class="tsd-kind-icon">draw<wbr>Boxes</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawlabels" class="tsd-kind-icon">draw<wbr>Labels</a></li>
@ -134,6 +135,13 @@
<aside class="tsd-sources">
</aside>
</section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="calculatehandbox" class="tsd-anchor"></a>
<h3>calculate<wbr>Hand<wbr>Box</h3>
<div class="tsd-signature tsd-kind-icon">calculate<wbr>Hand<wbr>Box<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Boolean</span></div>
<aside class="tsd-sources">
</aside>
</section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="color" class="tsd-anchor"></a>
<h3>color</h3>
@ -275,6 +283,9 @@
<li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="drawoptions.html#bufferedoutput" class="tsd-kind-icon">buffered<wbr>Output</a>
</li>
<li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="drawoptions.html#calculatehandbox" class="tsd-kind-icon">calculate<wbr>Hand<wbr>Box</a>
</li>
<li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="drawoptions.html#color" class="tsd-kind-icon">color</a>
</li>

View File

@ -37,6 +37,7 @@ export interface DrawOptions {
useCurves: Boolean;
bufferedOutput: Boolean;
useRawBoxes: Boolean;
calculateHandBox: Boolean;
}
export declare const options: DrawOptions;
export declare function gesture(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions): Promise<void>;