ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

mobile demo optimization and iris gestures

pull/293/head
Vladimir Mandic 2021-04-18 19:33:40 -04:00
parent 5de743ceb2
commit 20f61a6b2b
8 changed files with 88 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGELOG.md
View File

@ -1,6 +1,6 @@
# @vladmandic/human
Version: **1.5.2**
Version: **1.6.0**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>**
@ -11,9 +11,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/04/16 mandic00@live.com
### **origin/main** 2021/04/16 mandic00@live.com
- full rebuild
- new look
- added benchmarks
- added node-multiprocess demo
- fix image orientation
- flat app style

16
demo/helpers/menu.js
View File

@ -118,7 +118,7 @@ class Menu {
this.menu.appendChild(this.container);
if (typeof parent === 'object') parent.appendChild(this.menu);
else document.getElementById(parent)?.appendChild(this.menu);
else document.getElementById(parent).appendChild(this.menu);
}
get newID() {
@ -131,11 +131,11 @@ class Menu {
}
get width() {
return this.menu?.offsetWidth || 0;
return this.menu.offsetWidth || 0;
}
get height() {
return this.menu?.offsetHeight || 0;
return this.menu.offsetHeight || 0;
}
hide() {
@ -203,8 +203,8 @@ class Menu {
el.innerHTML = `<div class="menu-checkbox"><input class="menu-checkbox" type="checkbox" id="${this.newID}" ${object[variable] ? 'checked' : ''}/><label class="menu-checkbox-label" for="${this.ID}"></label></div>${title}`;
if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => {
object[variable] = evt.target?.checked;
if (callback) callback(evt.target?.checked);
object[variable] = evt.target.checked;
if (callback) callback(evt.target.checked);
});
return el;
}
@ -223,7 +223,7 @@ class Menu {
el.style.fontVariant = document.body.style.fontVariant;
if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => {
if (callback) callback(items[evt.target?.selectedIndex]);
if (callback) callback(items[evt.target.selectedIndex]);
});
return el;
}
@ -235,9 +235,9 @@ class Menu {
if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => {
if (evt.target) {
object[variable] = parseInt(evt.target?.value) === parseFloat(evt.target?.value) ? parseInt(evt.target?.value) : parseFloat(evt.target?.value);
object[variable] = parseInt(evt.target.value) === parseFloat(evt.target.value) ? parseInt(evt.target.value) : parseFloat(evt.target.value);
evt.target.setAttribute('value', evt.target.value);
if (callback) callback(evt.target?.value);
if (callback) callback(evt.target.value);
}
});
el.input = el.children[0];

25
demo/helpers/webrtc.js
View File

@ -17,18 +17,21 @@ async function webRTC(server, streamName, elementName) {
const connection = new RTCPeerConnection();
connection.oniceconnectionstatechange = () => log('connection', connection.iceConnectionState);
connection.onnegotiationneeded = async () => {
const offer = await connection.createOffer();
await connection.setLocalDescription(offer);
const res = await fetch(`${server}/stream/receiver/${suuid}`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' },
body: new URLSearchParams({
suuid: `${suuid}`,
data: `${btoa(connection.localDescription?.sdp || '')}`,
}),
});
let offer;
if (connection.localDescription) {
offer = await connection.createOffer();
await connection.setLocalDescription(offer);
const res = await fetch(`${server}/stream/receiver/${suuid}`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' },
body: new URLSearchParams({
suuid: `${suuid}`,
data: `${btoa(connection.localDescription.sdp || '')}`,
}),
});
}
const data = res && res.ok ? await res.text() : '';
if (data.length === 0) {
if (data.length === 0 || !offer) {
log('cannot connect:', server);
} else {
connection.setRemoteDescription(new RTCSessionDescription({

4
demo/index.html
View File

@ -16,7 +16,7 @@
<style>
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../assets/lato-light.woff2') }
html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
body { margin: 0; background: black; color: white; overflow-x: hidden }
body { margin: 0; background: black; color: white; overflow-x: hidden; width: 100vw; height: 100vh; }
body::-webkit-scrollbar { display: none; }
hr { width: 100%; }
.play { position: absolute; width: 256px; height: 256px; z-index: 9; bottom: 15%; left: 50%; margin-left: -125px; display: none; filter: grayscale(1); }
@ -28,7 +28,7 @@
.status { position: absolute; width: 100vw; bottom: 10%; text-align: center; font-size: 4rem; font-weight: 100; text-shadow: 2px 2px #303030; }
.thumbnail { margin: 8px; box-shadow: 0 0 4px 4px dimgrey; }
.thumbnail:hover { box-shadow: 0 0 8px 8px dimgrey; filter: grayscale(1); }
.log { position: absolute; bottom: 0; margin: 0.4rem; font-size: 0.9rem; }
.log { position: absolute; bottom: 0; margin: 0.4rem 0.4rem 0 0.4rem; font-size: 0.9rem; }
.menubar { width: 100vw; background: #303030; display: flex; justify-content: space-evenly; text-align: center; padding: 8px; cursor: pointer; }
.samples-container { display: flex; flex-wrap: wrap; }
.video { display: none; }

42
demo/index.js
View File

@ -101,7 +101,8 @@ const compare = { enabled: false, original: null };
async function calcSimmilariry(result) {
document.getElementById('compare-container').style.display = compare.enabled ? 'block' : 'none';
if (!compare.enabled) return;
if (!(result?.face?.length > 0) || (result?.face[0]?.embedding?.length <= 64)) return;
if (!result || !result.face || result.face[0].embedding) return;
if (!(result.face.length > 0) || (result.face[0].embedding.length <= 64)) return;
if (!compare.original) {
compare.original = result;
log('setting face compare baseline:', result.face[0]);
@ -120,7 +121,7 @@ async function calcSimmilariry(result) {
document.getElementById('compare-canvas').getContext('2d').drawImage(compare.original.canvas, 0, 0, 200, 200);
}
}
const similarity = human.similarity(compare.original?.face[0]?.embedding, result?.face[0]?.embedding);
const similarity = human.similarity(compare.original.face[0].embedding, result.face[0].embedding);
document.getElementById('similarity').innerText = `similarity: ${Math.trunc(1000 * similarity) / 10}%`;
}
@ -250,7 +251,7 @@ async function setupCamera() {
const track = stream.getVideoTracks()[0];
const settings = track.getSettings();
// log('camera constraints:', constraints, 'window:', { width: window.innerWidth, height: window.innerHeight }, 'settings:', settings, 'track:', track);
ui.camera = { name: track.label?.toLowerCase(), width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
ui.camera = { name: track.label.toLowerCase(), width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
return new Promise((resolve) => {
video.onloadeddata = async () => {
video.width = video.videoWidth;
@ -327,7 +328,7 @@ function runHumanDetect(input, canvas, timestamp) {
// if we want to continue and camera not ready, retry in 0.5sec, else just give up
if (input.paused) log('camera paused');
else if ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500);
else log(`camera not ready: track state: ${input.srcObject?.getVideoTracks()[0].readyState} stream state: ${input.readyState}`);
else log(`camera not ready: track state: ${input.srcObject.getVideoTracks()[0].readyState} stream state: ${input.readyState}`);
clearTimeout(ui.drawThread);
ui.drawThread = null;
log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw);
@ -442,11 +443,12 @@ async function detectSampleImages() {
function setupMenu() {
const x = [`${document.getElementById('btnDisplay').offsetLeft}px`, `${document.getElementById('btnImage').offsetLeft}px`, `${document.getElementById('btnProcess').offsetLeft}px`, `${document.getElementById('btnModel').offsetLeft}px`];
const top = `${document.getElementById('menubar').offsetHeight - 3}px`;
const top = `${document.getElementById('menubar').clientHeight}px`;
menu.display = new Menu(document.body, '', { top, left: x[0] });
menu.display.addBool('perf monitor', ui, 'bench', (val) => ui.bench = val);
menu.display.addBool('buffered output', ui, 'buffered', (val) => ui.buffered = val);
menu.display.addBool('buffer output', ui, 'buffered', (val) => ui.buffered = val);
menu.display.addBool('crop & scale', ui, 'crop', (val) => {
ui.crop = val;
setupCamera();
@ -456,8 +458,8 @@ function setupMenu() {
setupCamera();
});
menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.display.addBool('use 3D depth', human.draw.options, 'useDepth');
menu.display.addBool('draw with curves', human.draw.options, 'useCurves');
menu.display.addBool('use depth', human.draw.options, 'useDepth');
menu.display.addBool('use curves', human.draw.options, 'useCurves');
menu.display.addBool('print labels', human.draw.options, 'drawLabels');
menu.display.addBool('draw points', human.draw.options, 'drawPoints');
menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes');
@ -557,17 +559,28 @@ function setupMenu() {
}
async function resize() {
window.onresize = null;
const viewportScale = Math.min(1, Math.round(100 * window.innerWidth / 960) / 100);
const viewport = document.querySelector('meta[name=viewport]');
viewport.setAttribute('content', `width=device-width, shrink-to-fit=yes, minimum-scale=0.2, maximum-scale=2.0, user-scalable=yes, initial-scale=${viewportScale}`);
const x = [`${document.getElementById('btnDisplay').offsetLeft}px`, `${document.getElementById('btnImage').offsetLeft}px`, `${document.getElementById('btnProcess').offsetLeft}px`, `${document.getElementById('btnModel').offsetLeft}px`];
const top = `${document.getElementById('menubar').clientHeight - 3}px`;
menu.display.menu.style.top = top;
menu.image.menu.style.top = top;
menu.process.menu.style.top = top;
menu.models.menu.style.top = top;
menu.display.menu.style.left = x[0];
menu.image.menu.style.left = x[1];
menu.process.menu.style.left = x[2];
menu.models.menu.style.left = x[3];
const viewportScale = Math.min(1, Math.round(100 * window.innerWidth / 960) / 100);
const viewport = document.querySelector('meta[name=viewport]');
viewport.setAttribute('content', `width=device-width, shrink-to-fit=yes, minimum-scale=0.2, maximum-scale=2.0, user-scalable=yes, initial-scale=${viewportScale}`);
// console.log('view', viewportScale, window.innerWidth, viewport);
// document.body.style.MozTransform = `scale(${viewportScale})`;
// document.body.style.zoom = `scale(${viewportScale})`;
const fontSize = Math.trunc(10 * (1 - viewportScale)) + 16;
document.documentElement.style.fontSize = `${fontSize}px`;
human.draw.options.font = `small-caps ${fontSize + 4}px "Segoe UI"`;
setupCamera();
}
@ -638,6 +651,7 @@ async function main() {
document.getElementById('loader').style.display = 'none';
document.getElementById('play').style.display = 'block';
log('demo ready...');
for (const m of Object.values(menu)) m.hide();
}
window.onload = main;

4
package.json
View File

@ -1,6 +1,6 @@
{
"name": "@vladmandic/human",
"version": "1.5.2",
"version": "1.6.0",
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
"sideEffects": false,
"main": "dist/human.node.js",
@ -68,7 +68,7 @@
"canvas": "^2.7.0",
"chokidar": "^3.5.1",
"dayjs": "^1.10.4",
"esbuild": "^0.11.11",
"esbuild": "^0.11.12",
"eslint": "^7.24.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.22.1",

26
src/draw/draw.ts
View File

@ -41,6 +41,7 @@ export interface DrawOptions {
useCurves: Boolean,
bufferedOutput: Boolean,
useRawBoxes: Boolean,
calculateHandBox: Boolean,
}
export const options: DrawOptions = {
@ -61,6 +62,7 @@ export const options: DrawOptions = {
useCurves: <Boolean>false,
bufferedOutput: <Boolean>false,
useRawBoxes: <Boolean>false,
calculateHandBox: <Boolean>true,
};
function point(ctx, x, y, z = 0, localOptions) {
@ -359,15 +361,31 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<any>, draw
if (localOptions.drawBoxes) {
ctx.strokeStyle = localOptions.color;
ctx.fillStyle = localOptions.color;
if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3], localOptions);
else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions);
let box;
if (!localOptions.calculateHandBox) {
box = localOptions.useRawBoxes ? h.boxRaw : h.box;
} else {
box = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0];
if (h.landmarks && h.landmarks.length > 0) {
for (const pt of h.landmarks) {
if (pt[0] < box[0]) box[0] = pt[0];
if (pt[1] < box[1]) box[1] = pt[1];
if (pt[0] > box[2]) box[2] = pt[0];
if (pt[1] > box[3]) box[3] = pt[1];
}
box[2] -= box[0];
box[3] -= box[1];
}
}
if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * box[0], inCanvas.height * box[1], inCanvas.width * box[2], inCanvas.height * box[3], localOptions);
else rect(ctx, box[0], box[1], box[2], box[3], localOptions);
if (localOptions.drawLabels) {
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
ctx.fillStyle = localOptions.shadowColor;
ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]);
ctx.fillText('hand', box[0] + 3, 1 + box[1] + localOptions.lineHeight, box[2]);
}
ctx.fillStyle = localOptions.labelColor;
ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]);
ctx.fillText('hand', box[0] + 2, 0 + box[1] + localOptions.lineHeight, box[2]);
}
ctx.stroke();
}

10
src/gesture/gesture.ts
View File

@ -24,7 +24,7 @@ export const face = (res) => {
for (let i = 0; i < res.length; i++) {
if (res[i].mesh && res[i].mesh.length > 0) {
const eyeFacing = res[i].mesh[33][2] - res[i].mesh[263][2];
if (Math.abs(eyeFacing) < 10) gestures.push({ face: i, gesture: 'facing camera' });
if (Math.abs(eyeFacing) < 10) gestures.push({ face: i, gesture: 'facing center' });
else gestures.push({ face: i, gesture: `facing ${eyeFacing < 0 ? 'right' : 'left'}` });
const openLeft = Math.abs(res[i].mesh[374][1] - res[i].mesh[386][1]) / Math.abs(res[i].mesh[443][1] - res[i].mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' });
@ -53,7 +53,13 @@ export const iris = (res) => {
const areaRight = Math.abs(sizeXRight * sizeYRight);
const difference = Math.abs(areaLeft - areaRight) / Math.max(areaLeft, areaRight);
if (difference < 0.25) gestures.push({ iris: i, gesture: 'looking at camera' });
if (difference < 0.25) gestures.push({ iris: i, gesture: 'facing center' });
const rightIrisCenterX = Math.abs(res[i].mesh[33][0] - res[i].annotations.rightEyeIris[0][0]) / res[i].annotations.rightEyeIris[0][0];
const leftIrisCenterX = Math.abs(res[i].mesh[263][0] - res[i].annotations.leftEyeIris[0][0]) / res[i].annotations.leftEyeIris[0][0];
if (leftIrisCenterX > 0.025 && rightIrisCenterX > 0.025) gestures.push({ iris: i, gesture: 'looking center' });
else if (leftIrisCenterX > 0.025) gestures.push({ iris: i, gesture: 'looking right' });
else if (rightIrisCenterX > 0.025) gestures.push({ iris: i, gesture: 'looking left' });
}
return gestures;
};