implemented service worker

pull/134/head
Vladimir Mandic 2021-05-30 17:56:40 -04:00
parent e0374f02df
commit f8104569ec
19 changed files with 1138 additions and 835 deletions

View File

@ -11,6 +11,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/05/30 mandic00@live.com
- quantized centernet
- release candidate
- added usage restrictions
- quantize handdetect model
- added experimental movenet-lightning and removed blazepose from default dist
- added experimental face.rotation.gaze

View File

@ -9,7 +9,7 @@
**AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition,**
**Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis,**
**Age & Gender & Emotion Prediction, Gesture Recognition**
**Age & Gender & Emotion Prediction, Gaze Tracking, Gesture Recognition**
<br>

View File

@ -17,7 +17,6 @@ N/A
## In Progress
- Face rotation interpolation
- Object detection interpolation
## Issues

138
demo/index-pwa.js Normal file
View File

@ -0,0 +1,138 @@
/**
* PWA Service Worker for Human main demo
*/
/// <reference lib="webworker" />
// // @ts-nocheck Linting of ServiceWorker is not supported for JS files
const skipCaching = false;
const cacheName = 'Human';
const cacheFiles = ['/favicon.ico', 'manifest.webmanifest']; // assets and models are cached on first access
let cacheModels = true; // *.bin; *.json
let cacheWASM = true; // *.wasm
let cacheOther = false; // *
let listening = false;
const stats = { hit: 0, miss: 0 };
const log = (...msg) => {
const dt = new Date();
const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`;
// eslint-disable-next-line no-console
console.log(ts, 'pwa', ...msg);
};
async function updateCached(req) {
fetch(req)
.then((update) => {
// update cache if request is ok
if (update.ok) {
caches
.open(cacheName)
.then((cache) => cache.put(req, update))
.catch((err) => log('cache update error', err));
}
return true;
})
.catch((err) => {
log('fetch error', err);
return false;
});
}
async function getCached(evt) {
// just fetch
if (skipCaching) return fetch(evt.request);
// get from cache or fetch if not in cache
let found = await caches.match(evt.request);
if (found && found.ok) {
stats.hit += 1;
} else {
stats.miss += 1;
found = await fetch(evt.request);
}
// if still don't have it, return offline page
if (!found || !found.ok) {
found = await caches.match('offline.html');
}
// update cache in the background
if (found && found.type === 'basic' && found.ok) {
const uri = new URL(evt.request.url);
if (uri.pathname.endsWith('.bin') || uri.pathname.endsWith('.json')) {
if (cacheModels) updateCached(evt.request);
} else if (uri.pathname.endsWith('.wasm')) {
if (cacheWASM) updateCached(evt.request);
} else if (cacheOther) {
updateCached(evt.request);
}
}
return found;
}
function cacheInit() {
// eslint-disable-next-line promise/catch-or-return
caches.open(cacheName)
// eslint-disable-next-line promise/no-nesting
.then((cache) => cache.addAll(cacheFiles)
.then(
() => log('cache refresh:', cacheFiles.length, 'files'),
(err) => log('cache error', err),
));
}
if (!listening) {
// get messages from main app to update configuration
self.addEventListener('message', (evt) => {
log('event message:', evt.data);
switch (evt.data.key) {
case 'cacheModels': cacheModels = evt.data.val; break;
case 'cacheWASM': cacheWASM = evt.data.val; break;
case 'cacheOther': cacheOther = evt.data.val; break;
default:
}
});
self.addEventListener('install', (evt) => {
log('install');
// @ts-ignore scope for self is ServiceWorkerGlobalScope not Window
self.skipWaiting();
evt.waitUntil(cacheInit);
});
self.addEventListener('activate', (evt) => {
log('activate');
// @ts-ignore scope for self is ServiceWorkerGlobalScope not Window
evt.waitUntil(self.clients.claim());
});
self.addEventListener('fetch', (evt) => {
const uri = new URL(evt.request.url);
// if (uri.pathname === '/') { log('cache skip /', evt.request); return; } // skip root access requests
if (evt.request.cache === 'only-if-cached' && evt.request.mode !== 'same-origin') return; // required due to chrome bug
if (uri.origin !== location.origin) return; // skip non-local requests
if (evt.request.method !== 'GET') return; // only cache get requests
if (evt.request.url.includes('/api/')) return; // don't cache api requests, failures are handled at the time of call
const response = getCached(evt);
if (response) evt.respondWith(response);
else log('fetch response missing');
});
// only trigger controllerchange once
let refreshed = false;
self.addEventListener('controllerchange', (evt) => {
log(`PWA: ${evt.type}`);
if (refreshed) return;
refreshed = true;
location.reload();
});
listening = true;
}

View File

@ -36,24 +36,25 @@ const userConfig = {
enabled: false,
flip: false,
},
face: { enabled: false,
face: { enabled: true,
detector: { return: true },
mesh: { enabled: true },
iris: { enabled: true },
description: { enabled: false },
emotion: { enabled: false },
description: { enabled: true },
emotion: { enabled: true },
},
hand: { enabled: false },
// body: { enabled: true, modelPath: 'posenet.json' },
// body: { enabled: true, modelPath: 'blazepose.json' },
body: { enabled: false, modelPath: 'movenet-lightning.json' },
object: { enabled: true },
object: { enabled: false },
gesture: { enabled: true },
};
const drawOptions = {
bufferedOutput: true, // experimental feature that makes draw functions interpolate results between each detection for smoother movement
bufferedFactor: 3, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
bufferedFactor: 4, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
drawGaze: true,
};
// ui options
@ -110,6 +111,15 @@ const ui = {
],
};
const pwa = {
enabled: true,
cacheName: 'Human',
scriptFile: 'index-pwa.js',
cacheModels: true,
cacheWASM: true,
cacheOther: false,
};
// global variables
const menu = {};
let worker;
@ -660,6 +670,43 @@ async function drawWarmup(res) {
await human.draw.all(canvas, res, drawOptions);
}
async function pwaRegister() {
if (!pwa.enabled) return;
if ('serviceWorker' in navigator) {
try {
let found;
const regs = await navigator.serviceWorker.getRegistrations();
for (const reg of regs) {
log('pwa found:', reg.scope);
if (reg.scope.startsWith(location.origin)) found = reg;
}
if (!found) {
const reg = await navigator.serviceWorker.register(pwa.scriptFile, { scope: '/' });
found = reg;
log('pwa registered:', reg.scope);
}
} catch (err) {
if (err.name === 'SecurityError') log('pwa: ssl certificate is untrusted');
else log('pwa error:', err);
}
if (navigator.serviceWorker.controller) {
// update pwa configuration as it doesn't have access to it
navigator.serviceWorker.controller.postMessage({ key: 'cacheModels', val: pwa.cacheModels });
navigator.serviceWorker.controller.postMessage({ key: 'cacheWASM', val: pwa.cacheWASM });
navigator.serviceWorker.controller.postMessage({ key: 'cacheOther', val: pwa.cacheOther });
log('pwa ctive:', navigator.serviceWorker.controller.scriptURL);
const cache = await caches.open(pwa.cacheName);
if (cache) {
const content = await cache.matchAll();
log('pwa cache:', content.length, 'files');
}
}
} else {
log('pwa inactive');
}
}
async function main() {
window.addEventListener('unhandledrejection', (evt) => {
// eslint-disable-next-line no-console
@ -679,6 +726,9 @@ async function main() {
log('workers are disabled due to missing browser functionality');
}
// register PWA ServiceWorker
await pwaRegister();
// parse url search params
const params = new URLSearchParams(location.search);
log('url options:', params.toString());
@ -735,10 +785,16 @@ async function main() {
for (const m of Object.values(menu)) m.hide();
if (params.has('image')) {
const image = JSON.parse(params.get('image'));
log('overriding image:', image);
ui.samples = [image];
await detectSampleImages();
try {
const image = JSON.parse(params.get('image'));
log('overriding image:', image);
ui.samples = [image];
} catch {
status('cannot parse input image');
log('cannot parse input image', params.get('image'));
ui.samples = [];
}
if (ui.samples.length > 0) await detectSampleImages();
}
if (params.has('images')) {

35
demo/offline.html Normal file
View File

@ -0,0 +1,35 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="content-type" content="text/html; charset=utf-8">
<title>Human: Offline</title>
<meta name="viewport" content="width=device-width, shrink-to-fit=yes">
<meta name="mobile-web-app-capable" content="yes">
<meta name="application-name" content="Human">
<meta name="keywords" content="Human">
<meta name="description" content="Human; Author: Vladimir Mandic <mandic00@live.com>">
<meta name="msapplication-tooltip" content="Human; Author: Vladimir Mandic <mandic00@live.com>">
<link rel="manifest" href="manifest.webmanifest">
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
<link rel="icon" sizes="256x256" href="../assets/icon.png">
<link rel="apple-touch-icon" href="../assets/icon.png">
<link rel="apple-touch-startup-image" href="../assets/icon.png">
<style>
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../assets/lato-light.woff2') }
body { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; background: black; color: #ebebeb; }
h1 { font-size: 2rem; margin-top: 1.2rem; font-weight: bold; }
a { color: white; }
a:link { color: lightblue; text-decoration: none; }
a:hover { color: lightskyblue; text-decoration: none; }
.row { width: 90vw; margin: auto; margin-top: 100px; text-align: center; }
</style>
</head>
<body>
<div class="row text-center">
<h1>
<a href="/">Human: Offline</a><br>
<img alt="icon" src="../assets/icon.png">
</h1>
</div>
</body>
</html>

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

776
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

776
dist/human.js vendored

File diff suppressed because one or more lines are too long

View File

@ -4326,8 +4326,8 @@ async function predict3(image15, config3, idx, count2) {
// src/face.ts
var calculateGaze = (mesh) => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]);
const offsetIris = [0, 0];
const eyeRatio = 5;
const offsetIris = [0, -0.1];
const eyeRatio = 1;
const left = mesh[33][2] > mesh[263][2];
const irisCenter = left ? mesh[473] : mesh[468];
const eyeCenter = left ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2];
@ -18492,6 +18492,8 @@ function process4(input, config3) {
} else {
const originalWidth = input["naturalWidth"] || input["videoWidth"] || input["width"] || input["shape"] && input["shape"][1] > 0;
const originalHeight = input["naturalHeight"] || input["videoHeight"] || input["height"] || input["shape"] && input["shape"][2] > 0;
if (!originalWidth || !originalHeight)
return { tensor: null, canvas: inCanvas };
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (targetWidth > maxSize) {
@ -18642,6 +18644,7 @@ var options = {
bufferedOutput: false
};
var bufferedResult = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
var rad2deg = (theta) => Math.round(theta * 180 / Math.PI);
function point(ctx, x, y, z = 0, localOptions) {
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.beginPath();
@ -18763,8 +18766,12 @@ async function face2(inCanvas2, result, drawOptions) {
const emotion2 = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
labels2.push(emotion2.join(" "));
}
if (f.rotation && f.rotation.angle && f.rotation.angle.roll)
labels2.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
if (f.rotation.angle.roll)
labels2.push(`roll: ${rad2deg(f.rotation.angle.roll)}\xB0 yaw:${rad2deg(f.rotation.angle.yaw)}\xB0 pitch:${rad2deg(f.rotation.angle.pitch)}\xB0`);
if (f.rotation.gaze.angle)
labels2.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}\xB0`);
}
if (labels2.length === 0)
labels2.push("face");
ctx.fillStyle = localOptions.color;
@ -19135,7 +19142,18 @@ function calcBuffered(newResult, localOptions) {
for (let i = 0; i < newResult.face.length; i++) {
const box6 = newResult.face[i].box.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
const boxRaw3 = newResult.face[i].boxRaw.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
bufferedResult.face[i] = { ...newResult.face[i], box: box6, boxRaw: boxRaw3 };
const matrix = newResult.face[i].rotation.matrix;
const angle = {
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor
};
const gaze = {
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor
};
const rotation = { angle, matrix, gaze };
bufferedResult.face[i] = { ...newResult.face[i], rotation, box: box6, boxRaw: boxRaw3 };
}
const newPersons = newResult.persons;
if (!bufferedResult.persons || newPersons.length !== bufferedResult.persons.length)

View File

@ -4327,8 +4327,8 @@ async function predict3(image15, config3, idx, count2) {
// src/face.ts
var calculateGaze = (mesh) => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]);
const offsetIris = [0, 0];
const eyeRatio = 5;
const offsetIris = [0, -0.1];
const eyeRatio = 1;
const left = mesh[33][2] > mesh[263][2];
const irisCenter = left ? mesh[473] : mesh[468];
const eyeCenter = left ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2];
@ -18493,6 +18493,8 @@ function process4(input, config3) {
} else {
const originalWidth = input["naturalWidth"] || input["videoWidth"] || input["width"] || input["shape"] && input["shape"][1] > 0;
const originalHeight = input["naturalHeight"] || input["videoHeight"] || input["height"] || input["shape"] && input["shape"][2] > 0;
if (!originalWidth || !originalHeight)
return { tensor: null, canvas: inCanvas };
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (targetWidth > maxSize) {
@ -18643,6 +18645,7 @@ var options = {
bufferedOutput: false
};
var bufferedResult = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
var rad2deg = (theta) => Math.round(theta * 180 / Math.PI);
function point(ctx, x, y, z = 0, localOptions) {
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.beginPath();
@ -18764,8 +18767,12 @@ async function face2(inCanvas2, result, drawOptions) {
const emotion2 = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
labels2.push(emotion2.join(" "));
}
if (f.rotation && f.rotation.angle && f.rotation.angle.roll)
labels2.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
if (f.rotation.angle.roll)
labels2.push(`roll: ${rad2deg(f.rotation.angle.roll)}\xB0 yaw:${rad2deg(f.rotation.angle.yaw)}\xB0 pitch:${rad2deg(f.rotation.angle.pitch)}\xB0`);
if (f.rotation.gaze.angle)
labels2.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}\xB0`);
}
if (labels2.length === 0)
labels2.push("face");
ctx.fillStyle = localOptions.color;
@ -19136,7 +19143,18 @@ function calcBuffered(newResult, localOptions) {
for (let i = 0; i < newResult.face.length; i++) {
const box6 = newResult.face[i].box.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
const boxRaw3 = newResult.face[i].boxRaw.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
bufferedResult.face[i] = { ...newResult.face[i], box: box6, boxRaw: boxRaw3 };
const matrix = newResult.face[i].rotation.matrix;
const angle = {
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor
};
const gaze = {
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor
};
const rotation = { angle, matrix, gaze };
bufferedResult.face[i] = { ...newResult.face[i], rotation, box: box6, boxRaw: boxRaw3 };
}
const newPersons = newResult.persons;
if (!bufferedResult.persons || newPersons.length !== bufferedResult.persons.length)

28
dist/human.node.js vendored
View File

@ -4326,8 +4326,8 @@ async function predict3(image15, config3, idx, count2) {
// src/face.ts
var calculateGaze = (mesh) => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]);
const offsetIris = [0, 0];
const eyeRatio = 5;
const offsetIris = [0, -0.1];
const eyeRatio = 1;
const left = mesh[33][2] > mesh[263][2];
const irisCenter = left ? mesh[473] : mesh[468];
const eyeCenter = left ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2];
@ -18492,6 +18492,8 @@ function process4(input, config3) {
} else {
const originalWidth = input["naturalWidth"] || input["videoWidth"] || input["width"] || input["shape"] && input["shape"][1] > 0;
const originalHeight = input["naturalHeight"] || input["videoHeight"] || input["height"] || input["shape"] && input["shape"][2] > 0;
if (!originalWidth || !originalHeight)
return { tensor: null, canvas: inCanvas };
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (targetWidth > maxSize) {
@ -18642,6 +18644,7 @@ var options = {
bufferedOutput: false
};
var bufferedResult = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
var rad2deg = (theta) => Math.round(theta * 180 / Math.PI);
function point(ctx, x, y, z = 0, localOptions) {
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.beginPath();
@ -18763,8 +18766,12 @@ async function face2(inCanvas2, result, drawOptions) {
const emotion2 = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
labels2.push(emotion2.join(" "));
}
if (f.rotation && f.rotation.angle && f.rotation.angle.roll)
labels2.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
if (f.rotation.angle.roll)
labels2.push(`roll: ${rad2deg(f.rotation.angle.roll)}\xB0 yaw:${rad2deg(f.rotation.angle.yaw)}\xB0 pitch:${rad2deg(f.rotation.angle.pitch)}\xB0`);
if (f.rotation.gaze.angle)
labels2.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}\xB0`);
}
if (labels2.length === 0)
labels2.push("face");
ctx.fillStyle = localOptions.color;
@ -19135,7 +19142,18 @@ function calcBuffered(newResult, localOptions) {
for (let i = 0; i < newResult.face.length; i++) {
const box6 = newResult.face[i].box.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
const boxRaw3 = newResult.face[i].boxRaw.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
bufferedResult.face[i] = { ...newResult.face[i], box: box6, boxRaw: boxRaw3 };
const matrix = newResult.face[i].rotation.matrix;
const angle = {
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor
};
const gaze = {
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor
};
const rotation = { angle, matrix, gaze };
bufferedResult.face[i] = { ...newResult.face[i], rotation, box: box6, boxRaw: boxRaw3 };
}
const newPersons = newResult.persons;
if (!bufferedResult.persons || newPersons.length !== bufferedResult.persons.length)

View File

@ -42,6 +42,7 @@
"emotion-detection",
"gender-prediction",
"gesture-recognition",
"gaze-tracking",
"age-gender",
"faceapi",
"face",

View File

@ -1,17 +1,17 @@
2021-05-30 12:02:00 INFO:  @vladmandic/human version 2.0.0
2021-05-30 12:02:00 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-05-30 12:02:00 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-05-30 12:02:00 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 12:02:00 STATE: Build for: node type: node: {"imports":39,"importBytes":444757,"outputBytes":396794,"outputFiles":"dist/human.node.js"}
2021-05-30 12:02:00 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 12:02:00 STATE: Build for: nodeGPU type: node: {"imports":39,"importBytes":444765,"outputBytes":396798,"outputFiles":"dist/human.node-gpu.js"}
2021-05-30 12:02:00 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 12:02:01 STATE: Build for: nodeWASM type: node: {"imports":39,"importBytes":444832,"outputBytes":396870,"outputFiles":"dist/human.node-wasm.js"}
2021-05-30 12:02:01 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 12:02:01 STATE: Build for: browserNoBundle type: esm: {"imports":39,"importBytes":444859,"outputBytes":242556,"outputFiles":"dist/human.esm-nobundle.js"}
2021-05-30 12:02:01 STATE: Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111418,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 12:02:02 STATE: Build for: browserBundle type: iife: {"imports":39,"importBytes":1554883,"outputBytes":1350365,"outputFiles":"dist/human.js"}
2021-05-30 12:02:02 STATE: Build for: browserBundle type: esm: {"imports":39,"importBytes":1554883,"outputBytes":1350357,"outputFiles":"dist/human.esm.js"}
2021-05-30 12:02:02 INFO:  Generate types: ["src/human.ts"]
2021-05-30 12:02:07 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-05-30 12:02:07 INFO:  Generate TypeDocs: ["src/human.ts"]
2021-05-30 17:55:58 INFO:  @vladmandic/human version 2.0.0
2021-05-30 17:55:58 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-05-30 17:55:58 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-05-30 17:55:58 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 17:55:58 STATE: Build for: node type: node: {"imports":39,"importBytes":446068,"outputBytes":398047,"outputFiles":"dist/human.node.js"}
2021-05-30 17:55:58 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 17:55:58 STATE: Build for: nodeGPU type: node: {"imports":39,"importBytes":446076,"outputBytes":398051,"outputFiles":"dist/human.node-gpu.js"}
2021-05-30 17:55:58 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 17:55:58 STATE: Build for: nodeWASM type: node: {"imports":39,"importBytes":446143,"outputBytes":398123,"outputFiles":"dist/human.node-wasm.js"}
2021-05-30 17:55:58 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 17:55:58 STATE: Build for: browserNoBundle type: esm: {"imports":39,"importBytes":446170,"outputBytes":243299,"outputFiles":"dist/human.esm-nobundle.js"}
2021-05-30 17:55:59 STATE: Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111418,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 17:55:59 STATE: Build for: browserBundle type: iife: {"imports":39,"importBytes":1556194,"outputBytes":1351116,"outputFiles":"dist/human.js"}
2021-05-30 17:56:00 STATE: Build for: browserBundle type: esm: {"imports":39,"importBytes":1556194,"outputBytes":1351108,"outputFiles":"dist/human.esm.js"}
2021-05-30 17:56:00 INFO:  Generate types: ["src/human.ts"]
2021-05-30 17:56:05 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-05-30 17:56:05 INFO:  Generate TypeDocs: ["src/human.ts"]

View File

@ -71,6 +71,8 @@ export const options: DrawOptions = {
let bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
const rad2deg = (theta) => Math.round((theta * 180) / Math.PI);
function point(ctx, x, y, z = 0, localOptions) {
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
ctx.beginPath();
@ -186,7 +188,10 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
labels.push(emotion.join(' '));
}
if (f.rotation && f.rotation.angle && f.rotation.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`);
if (f.rotation.gaze.angle) labels.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}°`);
}
if (labels.length === 0) labels.push('face');
ctx.fillStyle = localOptions.color;
for (let i = labels.length - 1; i >= 0; i--) {
@ -553,7 +558,18 @@ function calcBuffered(newResult, localOptions) {
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
const boxRaw = newResult.face[i].boxRaw // update boxRaw
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
bufferedResult.face[i] = { ...newResult.face[i], box, boxRaw }; // shallow clone plus updated values
const matrix = newResult.face[i].rotation.matrix;
const angle = {
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor,
};
const gaze = {
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor,
};
const rotation = { angle, matrix, gaze };
bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values
}
// interpolate person results

View File

@ -15,8 +15,8 @@ const rad2deg = (theta) => (theta * 180) / Math.PI;
const calculateGaze = (mesh): { angle: number, strength: number } => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points
const offsetIris = [0, 0]; // tbd: iris center may not align with average of eye extremes
const eyeRatio = 5; // factor to normalize changes x vs y
const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes
const eyeRatio = 1; // factor to normalize changes x vs y
const left = mesh[33][2] > mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis
const irisCenter = left ? mesh[473] : mesh[468];

View File

@ -41,6 +41,7 @@ export function process(input, config): { tensor: Tensor | null, canvas: Offscre
// check if resizing will be needed
const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0));
const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0));
if (!originalWidth || !originalHeight) return { tensor: null, canvas: inCanvas }; // video may become temporarily unavailable due to onresize
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (targetWidth > maxSize) {