mirror of https://github.com/vladmandic/human
implemented service worker
parent
e0374f02df
commit
f8104569ec
|
@ -11,6 +11,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
### **HEAD -> main** 2021/05/30 mandic00@live.com
|
||||
|
||||
- quantized centernet
|
||||
- release candidate
|
||||
- added usage restrictions
|
||||
- quantize handdetect model
|
||||
- added experimental movenet-lightning and removed blazepose from default dist
|
||||
- added experimental face.rotation.gaze
|
||||
|
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
**AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition,**
|
||||
**Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis,**
|
||||
**Age & Gender & Emotion Prediction, Gesture Recognition**
|
||||
**Age & Gender & Emotion Prediction, Gaze Tracking, Gesture Recognition**
|
||||
|
||||
<br>
|
||||
|
||||
|
|
1
TODO.md
1
TODO.md
|
@ -17,7 +17,6 @@ N/A
|
|||
|
||||
## In Progress
|
||||
|
||||
- Face rotation interpolation
|
||||
- Object detection interpolation
|
||||
|
||||
## Issues
|
||||
|
|
|
@ -0,0 +1,138 @@
|
|||
/**
|
||||
* PWA Service Worker for Human main demo
|
||||
*/
|
||||
|
||||
/// <reference lib="webworker" />
|
||||
|
||||
// // @ts-nocheck Linting of ServiceWorker is not supported for JS files
|
||||
|
||||
const skipCaching = false;
|
||||
|
||||
const cacheName = 'Human';
|
||||
const cacheFiles = ['/favicon.ico', 'manifest.webmanifest']; // assets and models are cached on first access
|
||||
|
||||
let cacheModels = true; // *.bin; *.json
|
||||
let cacheWASM = true; // *.wasm
|
||||
let cacheOther = false; // *
|
||||
|
||||
let listening = false;
|
||||
const stats = { hit: 0, miss: 0 };
|
||||
|
||||
const log = (...msg) => {
|
||||
const dt = new Date();
|
||||
const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`;
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(ts, 'pwa', ...msg);
|
||||
};
|
||||
|
||||
async function updateCached(req) {
|
||||
fetch(req)
|
||||
.then((update) => {
|
||||
// update cache if request is ok
|
||||
if (update.ok) {
|
||||
caches
|
||||
.open(cacheName)
|
||||
.then((cache) => cache.put(req, update))
|
||||
.catch((err) => log('cache update error', err));
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.catch((err) => {
|
||||
log('fetch error', err);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
async function getCached(evt) {
|
||||
// just fetch
|
||||
if (skipCaching) return fetch(evt.request);
|
||||
|
||||
// get from cache or fetch if not in cache
|
||||
let found = await caches.match(evt.request);
|
||||
if (found && found.ok) {
|
||||
stats.hit += 1;
|
||||
} else {
|
||||
stats.miss += 1;
|
||||
found = await fetch(evt.request);
|
||||
}
|
||||
|
||||
// if still don't have it, return offline page
|
||||
if (!found || !found.ok) {
|
||||
found = await caches.match('offline.html');
|
||||
}
|
||||
|
||||
// update cache in the background
|
||||
if (found && found.type === 'basic' && found.ok) {
|
||||
const uri = new URL(evt.request.url);
|
||||
if (uri.pathname.endsWith('.bin') || uri.pathname.endsWith('.json')) {
|
||||
if (cacheModels) updateCached(evt.request);
|
||||
} else if (uri.pathname.endsWith('.wasm')) {
|
||||
if (cacheWASM) updateCached(evt.request);
|
||||
} else if (cacheOther) {
|
||||
updateCached(evt.request);
|
||||
}
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
function cacheInit() {
|
||||
// eslint-disable-next-line promise/catch-or-return
|
||||
caches.open(cacheName)
|
||||
// eslint-disable-next-line promise/no-nesting
|
||||
.then((cache) => cache.addAll(cacheFiles)
|
||||
.then(
|
||||
() => log('cache refresh:', cacheFiles.length, 'files'),
|
||||
(err) => log('cache error', err),
|
||||
));
|
||||
}
|
||||
|
||||
if (!listening) {
|
||||
// get messages from main app to update configuration
|
||||
self.addEventListener('message', (evt) => {
|
||||
log('event message:', evt.data);
|
||||
switch (evt.data.key) {
|
||||
case 'cacheModels': cacheModels = evt.data.val; break;
|
||||
case 'cacheWASM': cacheWASM = evt.data.val; break;
|
||||
case 'cacheOther': cacheOther = evt.data.val; break;
|
||||
default:
|
||||
}
|
||||
});
|
||||
|
||||
self.addEventListener('install', (evt) => {
|
||||
log('install');
|
||||
// @ts-ignore scope for self is ServiceWorkerGlobalScope not Window
|
||||
self.skipWaiting();
|
||||
evt.waitUntil(cacheInit);
|
||||
});
|
||||
|
||||
self.addEventListener('activate', (evt) => {
|
||||
log('activate');
|
||||
// @ts-ignore scope for self is ServiceWorkerGlobalScope not Window
|
||||
evt.waitUntil(self.clients.claim());
|
||||
});
|
||||
|
||||
self.addEventListener('fetch', (evt) => {
|
||||
const uri = new URL(evt.request.url);
|
||||
// if (uri.pathname === '/') { log('cache skip /', evt.request); return; } // skip root access requests
|
||||
if (evt.request.cache === 'only-if-cached' && evt.request.mode !== 'same-origin') return; // required due to chrome bug
|
||||
if (uri.origin !== location.origin) return; // skip non-local requests
|
||||
if (evt.request.method !== 'GET') return; // only cache get requests
|
||||
if (evt.request.url.includes('/api/')) return; // don't cache api requests, failures are handled at the time of call
|
||||
|
||||
const response = getCached(evt);
|
||||
if (response) evt.respondWith(response);
|
||||
else log('fetch response missing');
|
||||
});
|
||||
|
||||
// only trigger controllerchange once
|
||||
let refreshed = false;
|
||||
self.addEventListener('controllerchange', (evt) => {
|
||||
log(`PWA: ${evt.type}`);
|
||||
if (refreshed) return;
|
||||
refreshed = true;
|
||||
location.reload();
|
||||
});
|
||||
|
||||
listening = true;
|
||||
}
|
|
@ -36,24 +36,25 @@ const userConfig = {
|
|||
enabled: false,
|
||||
flip: false,
|
||||
},
|
||||
face: { enabled: false,
|
||||
face: { enabled: true,
|
||||
detector: { return: true },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
description: { enabled: false },
|
||||
emotion: { enabled: false },
|
||||
description: { enabled: true },
|
||||
emotion: { enabled: true },
|
||||
},
|
||||
hand: { enabled: false },
|
||||
// body: { enabled: true, modelPath: 'posenet.json' },
|
||||
// body: { enabled: true, modelPath: 'blazepose.json' },
|
||||
body: { enabled: false, modelPath: 'movenet-lightning.json' },
|
||||
object: { enabled: true },
|
||||
object: { enabled: false },
|
||||
gesture: { enabled: true },
|
||||
};
|
||||
|
||||
const drawOptions = {
|
||||
bufferedOutput: true, // experimental feature that makes draw functions interpolate results between each detection for smoother movement
|
||||
bufferedFactor: 3, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
|
||||
bufferedFactor: 4, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
|
||||
drawGaze: true,
|
||||
};
|
||||
|
||||
// ui options
|
||||
|
@ -110,6 +111,15 @@ const ui = {
|
|||
],
|
||||
};
|
||||
|
||||
const pwa = {
|
||||
enabled: true,
|
||||
cacheName: 'Human',
|
||||
scriptFile: 'index-pwa.js',
|
||||
cacheModels: true,
|
||||
cacheWASM: true,
|
||||
cacheOther: false,
|
||||
};
|
||||
|
||||
// global variables
|
||||
const menu = {};
|
||||
let worker;
|
||||
|
@ -660,6 +670,43 @@ async function drawWarmup(res) {
|
|||
await human.draw.all(canvas, res, drawOptions);
|
||||
}
|
||||
|
||||
async function pwaRegister() {
|
||||
if (!pwa.enabled) return;
|
||||
if ('serviceWorker' in navigator) {
|
||||
try {
|
||||
let found;
|
||||
const regs = await navigator.serviceWorker.getRegistrations();
|
||||
for (const reg of regs) {
|
||||
log('pwa found:', reg.scope);
|
||||
if (reg.scope.startsWith(location.origin)) found = reg;
|
||||
}
|
||||
if (!found) {
|
||||
const reg = await navigator.serviceWorker.register(pwa.scriptFile, { scope: '/' });
|
||||
found = reg;
|
||||
log('pwa registered:', reg.scope);
|
||||
}
|
||||
} catch (err) {
|
||||
if (err.name === 'SecurityError') log('pwa: ssl certificate is untrusted');
|
||||
else log('pwa error:', err);
|
||||
}
|
||||
if (navigator.serviceWorker.controller) {
|
||||
// update pwa configuration as it doesn't have access to it
|
||||
navigator.serviceWorker.controller.postMessage({ key: 'cacheModels', val: pwa.cacheModels });
|
||||
navigator.serviceWorker.controller.postMessage({ key: 'cacheWASM', val: pwa.cacheWASM });
|
||||
navigator.serviceWorker.controller.postMessage({ key: 'cacheOther', val: pwa.cacheOther });
|
||||
|
||||
log('pwa ctive:', navigator.serviceWorker.controller.scriptURL);
|
||||
const cache = await caches.open(pwa.cacheName);
|
||||
if (cache) {
|
||||
const content = await cache.matchAll();
|
||||
log('pwa cache:', content.length, 'files');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log('pwa inactive');
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
window.addEventListener('unhandledrejection', (evt) => {
|
||||
// eslint-disable-next-line no-console
|
||||
|
@ -679,6 +726,9 @@ async function main() {
|
|||
log('workers are disabled due to missing browser functionality');
|
||||
}
|
||||
|
||||
// register PWA ServiceWorker
|
||||
await pwaRegister();
|
||||
|
||||
// parse url search params
|
||||
const params = new URLSearchParams(location.search);
|
||||
log('url options:', params.toString());
|
||||
|
@ -735,10 +785,16 @@ async function main() {
|
|||
for (const m of Object.values(menu)) m.hide();
|
||||
|
||||
if (params.has('image')) {
|
||||
const image = JSON.parse(params.get('image'));
|
||||
log('overriding image:', image);
|
||||
ui.samples = [image];
|
||||
await detectSampleImages();
|
||||
try {
|
||||
const image = JSON.parse(params.get('image'));
|
||||
log('overriding image:', image);
|
||||
ui.samples = [image];
|
||||
} catch {
|
||||
status('cannot parse input image');
|
||||
log('cannot parse input image', params.get('image'));
|
||||
ui.samples = [];
|
||||
}
|
||||
if (ui.samples.length > 0) await detectSampleImages();
|
||||
}
|
||||
|
||||
if (params.has('images')) {
|
||||
|
|
|
@ -0,0 +1,35 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="content-type" content="text/html; charset=utf-8">
|
||||
<title>Human: Offline</title>
|
||||
<meta name="viewport" content="width=device-width, shrink-to-fit=yes">
|
||||
<meta name="mobile-web-app-capable" content="yes">
|
||||
<meta name="application-name" content="Human">
|
||||
<meta name="keywords" content="Human">
|
||||
<meta name="description" content="Human; Author: Vladimir Mandic <mandic00@live.com>">
|
||||
<meta name="msapplication-tooltip" content="Human; Author: Vladimir Mandic <mandic00@live.com>">
|
||||
<link rel="manifest" href="manifest.webmanifest">
|
||||
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
|
||||
<link rel="icon" sizes="256x256" href="../assets/icon.png">
|
||||
<link rel="apple-touch-icon" href="../assets/icon.png">
|
||||
<link rel="apple-touch-startup-image" href="../assets/icon.png">
|
||||
<style>
|
||||
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../assets/lato-light.woff2') }
|
||||
body { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; background: black; color: #ebebeb; }
|
||||
h1 { font-size: 2rem; margin-top: 1.2rem; font-weight: bold; }
|
||||
a { color: white; }
|
||||
a:link { color: lightblue; text-decoration: none; }
|
||||
a:hover { color: lightskyblue; text-decoration: none; }
|
||||
.row { width: 90vw; margin: auto; margin-top: 100px; text-align: center; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="row text-center">
|
||||
<h1>
|
||||
<a href="/">Human: Offline</a><br>
|
||||
<img alt="icon" src="../assets/icon.png">
|
||||
</h1>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
@ -4326,8 +4326,8 @@ async function predict3(image15, config3, idx, count2) {
|
|||
// src/face.ts
|
||||
var calculateGaze = (mesh) => {
|
||||
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]);
|
||||
const offsetIris = [0, 0];
|
||||
const eyeRatio = 5;
|
||||
const offsetIris = [0, -0.1];
|
||||
const eyeRatio = 1;
|
||||
const left = mesh[33][2] > mesh[263][2];
|
||||
const irisCenter = left ? mesh[473] : mesh[468];
|
||||
const eyeCenter = left ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2];
|
||||
|
@ -18492,6 +18492,8 @@ function process4(input, config3) {
|
|||
} else {
|
||||
const originalWidth = input["naturalWidth"] || input["videoWidth"] || input["width"] || input["shape"] && input["shape"][1] > 0;
|
||||
const originalHeight = input["naturalHeight"] || input["videoHeight"] || input["height"] || input["shape"] && input["shape"][2] > 0;
|
||||
if (!originalWidth || !originalHeight)
|
||||
return { tensor: null, canvas: inCanvas };
|
||||
let targetWidth = originalWidth;
|
||||
let targetHeight = originalHeight;
|
||||
if (targetWidth > maxSize) {
|
||||
|
@ -18642,6 +18644,7 @@ var options = {
|
|||
bufferedOutput: false
|
||||
};
|
||||
var bufferedResult = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
|
||||
var rad2deg = (theta) => Math.round(theta * 180 / Math.PI);
|
||||
function point(ctx, x, y, z = 0, localOptions) {
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.beginPath();
|
||||
|
@ -18763,8 +18766,12 @@ async function face2(inCanvas2, result, drawOptions) {
|
|||
const emotion2 = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
|
||||
labels2.push(emotion2.join(" "));
|
||||
}
|
||||
if (f.rotation && f.rotation.angle && f.rotation.angle.roll)
|
||||
labels2.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
|
||||
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
|
||||
if (f.rotation.angle.roll)
|
||||
labels2.push(`roll: ${rad2deg(f.rotation.angle.roll)}\xB0 yaw:${rad2deg(f.rotation.angle.yaw)}\xB0 pitch:${rad2deg(f.rotation.angle.pitch)}\xB0`);
|
||||
if (f.rotation.gaze.angle)
|
||||
labels2.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}\xB0`);
|
||||
}
|
||||
if (labels2.length === 0)
|
||||
labels2.push("face");
|
||||
ctx.fillStyle = localOptions.color;
|
||||
|
@ -19135,7 +19142,18 @@ function calcBuffered(newResult, localOptions) {
|
|||
for (let i = 0; i < newResult.face.length; i++) {
|
||||
const box6 = newResult.face[i].box.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
|
||||
const boxRaw3 = newResult.face[i].boxRaw.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
|
||||
bufferedResult.face[i] = { ...newResult.face[i], box: box6, boxRaw: boxRaw3 };
|
||||
const matrix = newResult.face[i].rotation.matrix;
|
||||
const angle = {
|
||||
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
|
||||
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
|
||||
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor
|
||||
};
|
||||
const gaze = {
|
||||
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
|
||||
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor
|
||||
};
|
||||
const rotation = { angle, matrix, gaze };
|
||||
bufferedResult.face[i] = { ...newResult.face[i], rotation, box: box6, boxRaw: boxRaw3 };
|
||||
}
|
||||
const newPersons = newResult.persons;
|
||||
if (!bufferedResult.persons || newPersons.length !== bufferedResult.persons.length)
|
||||
|
|
|
@ -4327,8 +4327,8 @@ async function predict3(image15, config3, idx, count2) {
|
|||
// src/face.ts
|
||||
var calculateGaze = (mesh) => {
|
||||
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]);
|
||||
const offsetIris = [0, 0];
|
||||
const eyeRatio = 5;
|
||||
const offsetIris = [0, -0.1];
|
||||
const eyeRatio = 1;
|
||||
const left = mesh[33][2] > mesh[263][2];
|
||||
const irisCenter = left ? mesh[473] : mesh[468];
|
||||
const eyeCenter = left ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2];
|
||||
|
@ -18493,6 +18493,8 @@ function process4(input, config3) {
|
|||
} else {
|
||||
const originalWidth = input["naturalWidth"] || input["videoWidth"] || input["width"] || input["shape"] && input["shape"][1] > 0;
|
||||
const originalHeight = input["naturalHeight"] || input["videoHeight"] || input["height"] || input["shape"] && input["shape"][2] > 0;
|
||||
if (!originalWidth || !originalHeight)
|
||||
return { tensor: null, canvas: inCanvas };
|
||||
let targetWidth = originalWidth;
|
||||
let targetHeight = originalHeight;
|
||||
if (targetWidth > maxSize) {
|
||||
|
@ -18643,6 +18645,7 @@ var options = {
|
|||
bufferedOutput: false
|
||||
};
|
||||
var bufferedResult = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
|
||||
var rad2deg = (theta) => Math.round(theta * 180 / Math.PI);
|
||||
function point(ctx, x, y, z = 0, localOptions) {
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.beginPath();
|
||||
|
@ -18764,8 +18767,12 @@ async function face2(inCanvas2, result, drawOptions) {
|
|||
const emotion2 = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
|
||||
labels2.push(emotion2.join(" "));
|
||||
}
|
||||
if (f.rotation && f.rotation.angle && f.rotation.angle.roll)
|
||||
labels2.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
|
||||
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
|
||||
if (f.rotation.angle.roll)
|
||||
labels2.push(`roll: ${rad2deg(f.rotation.angle.roll)}\xB0 yaw:${rad2deg(f.rotation.angle.yaw)}\xB0 pitch:${rad2deg(f.rotation.angle.pitch)}\xB0`);
|
||||
if (f.rotation.gaze.angle)
|
||||
labels2.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}\xB0`);
|
||||
}
|
||||
if (labels2.length === 0)
|
||||
labels2.push("face");
|
||||
ctx.fillStyle = localOptions.color;
|
||||
|
@ -19136,7 +19143,18 @@ function calcBuffered(newResult, localOptions) {
|
|||
for (let i = 0; i < newResult.face.length; i++) {
|
||||
const box6 = newResult.face[i].box.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
|
||||
const boxRaw3 = newResult.face[i].boxRaw.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
|
||||
bufferedResult.face[i] = { ...newResult.face[i], box: box6, boxRaw: boxRaw3 };
|
||||
const matrix = newResult.face[i].rotation.matrix;
|
||||
const angle = {
|
||||
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
|
||||
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
|
||||
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor
|
||||
};
|
||||
const gaze = {
|
||||
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
|
||||
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor
|
||||
};
|
||||
const rotation = { angle, matrix, gaze };
|
||||
bufferedResult.face[i] = { ...newResult.face[i], rotation, box: box6, boxRaw: boxRaw3 };
|
||||
}
|
||||
const newPersons = newResult.persons;
|
||||
if (!bufferedResult.persons || newPersons.length !== bufferedResult.persons.length)
|
||||
|
|
|
@ -4326,8 +4326,8 @@ async function predict3(image15, config3, idx, count2) {
|
|||
// src/face.ts
|
||||
var calculateGaze = (mesh) => {
|
||||
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]);
|
||||
const offsetIris = [0, 0];
|
||||
const eyeRatio = 5;
|
||||
const offsetIris = [0, -0.1];
|
||||
const eyeRatio = 1;
|
||||
const left = mesh[33][2] > mesh[263][2];
|
||||
const irisCenter = left ? mesh[473] : mesh[468];
|
||||
const eyeCenter = left ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2];
|
||||
|
@ -18492,6 +18492,8 @@ function process4(input, config3) {
|
|||
} else {
|
||||
const originalWidth = input["naturalWidth"] || input["videoWidth"] || input["width"] || input["shape"] && input["shape"][1] > 0;
|
||||
const originalHeight = input["naturalHeight"] || input["videoHeight"] || input["height"] || input["shape"] && input["shape"][2] > 0;
|
||||
if (!originalWidth || !originalHeight)
|
||||
return { tensor: null, canvas: inCanvas };
|
||||
let targetWidth = originalWidth;
|
||||
let targetHeight = originalHeight;
|
||||
if (targetWidth > maxSize) {
|
||||
|
@ -18642,6 +18644,7 @@ var options = {
|
|||
bufferedOutput: false
|
||||
};
|
||||
var bufferedResult = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
|
||||
var rad2deg = (theta) => Math.round(theta * 180 / Math.PI);
|
||||
function point(ctx, x, y, z = 0, localOptions) {
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.beginPath();
|
||||
|
@ -18763,8 +18766,12 @@ async function face2(inCanvas2, result, drawOptions) {
|
|||
const emotion2 = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
|
||||
labels2.push(emotion2.join(" "));
|
||||
}
|
||||
if (f.rotation && f.rotation.angle && f.rotation.angle.roll)
|
||||
labels2.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
|
||||
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
|
||||
if (f.rotation.angle.roll)
|
||||
labels2.push(`roll: ${rad2deg(f.rotation.angle.roll)}\xB0 yaw:${rad2deg(f.rotation.angle.yaw)}\xB0 pitch:${rad2deg(f.rotation.angle.pitch)}\xB0`);
|
||||
if (f.rotation.gaze.angle)
|
||||
labels2.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}\xB0`);
|
||||
}
|
||||
if (labels2.length === 0)
|
||||
labels2.push("face");
|
||||
ctx.fillStyle = localOptions.color;
|
||||
|
@ -19135,7 +19142,18 @@ function calcBuffered(newResult, localOptions) {
|
|||
for (let i = 0; i < newResult.face.length; i++) {
|
||||
const box6 = newResult.face[i].box.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
|
||||
const boxRaw3 = newResult.face[i].boxRaw.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
|
||||
bufferedResult.face[i] = { ...newResult.face[i], box: box6, boxRaw: boxRaw3 };
|
||||
const matrix = newResult.face[i].rotation.matrix;
|
||||
const angle = {
|
||||
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
|
||||
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
|
||||
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor
|
||||
};
|
||||
const gaze = {
|
||||
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
|
||||
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor
|
||||
};
|
||||
const rotation = { angle, matrix, gaze };
|
||||
bufferedResult.face[i] = { ...newResult.face[i], rotation, box: box6, boxRaw: boxRaw3 };
|
||||
}
|
||||
const newPersons = newResult.persons;
|
||||
if (!bufferedResult.persons || newPersons.length !== bufferedResult.persons.length)
|
||||
|
|
|
@ -42,6 +42,7 @@
|
|||
"emotion-detection",
|
||||
"gender-prediction",
|
||||
"gesture-recognition",
|
||||
"gaze-tracking",
|
||||
"age-gender",
|
||||
"faceapi",
|
||||
"face",
|
||||
|
|
|
@ -1,17 +1,17 @@
|
|||
2021-05-30 12:02:00 [36mINFO: [39m @vladmandic/human version 2.0.0
|
||||
2021-05-30 12:02:00 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.0.0
|
||||
2021-05-30 12:02:00 [36mINFO: [39m Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
|
||||
2021-05-30 12:02:00 [35mSTATE:[39m Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 12:02:00 [35mSTATE:[39m Build for: node type: node: {"imports":39,"importBytes":444757,"outputBytes":396794,"outputFiles":"dist/human.node.js"}
|
||||
2021-05-30 12:02:00 [35mSTATE:[39m Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 12:02:00 [35mSTATE:[39m Build for: nodeGPU type: node: {"imports":39,"importBytes":444765,"outputBytes":396798,"outputFiles":"dist/human.node-gpu.js"}
|
||||
2021-05-30 12:02:00 [35mSTATE:[39m Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 12:02:01 [35mSTATE:[39m Build for: nodeWASM type: node: {"imports":39,"importBytes":444832,"outputBytes":396870,"outputFiles":"dist/human.node-wasm.js"}
|
||||
2021-05-30 12:02:01 [35mSTATE:[39m Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 12:02:01 [35mSTATE:[39m Build for: browserNoBundle type: esm: {"imports":39,"importBytes":444859,"outputBytes":242556,"outputFiles":"dist/human.esm-nobundle.js"}
|
||||
2021-05-30 12:02:01 [35mSTATE:[39m Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111418,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 12:02:02 [35mSTATE:[39m Build for: browserBundle type: iife: {"imports":39,"importBytes":1554883,"outputBytes":1350365,"outputFiles":"dist/human.js"}
|
||||
2021-05-30 12:02:02 [35mSTATE:[39m Build for: browserBundle type: esm: {"imports":39,"importBytes":1554883,"outputBytes":1350357,"outputFiles":"dist/human.esm.js"}
|
||||
2021-05-30 12:02:02 [36mINFO: [39m Generate types: ["src/human.ts"]
|
||||
2021-05-30 12:02:07 [36mINFO: [39m Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
|
||||
2021-05-30 12:02:07 [36mINFO: [39m Generate TypeDocs: ["src/human.ts"]
|
||||
2021-05-30 17:55:58 [36mINFO: [39m @vladmandic/human version 2.0.0
|
||||
2021-05-30 17:55:58 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.0.0
|
||||
2021-05-30 17:55:58 [36mINFO: [39m Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
|
||||
2021-05-30 17:55:58 [35mSTATE:[39m Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 17:55:58 [35mSTATE:[39m Build for: node type: node: {"imports":39,"importBytes":446068,"outputBytes":398047,"outputFiles":"dist/human.node.js"}
|
||||
2021-05-30 17:55:58 [35mSTATE:[39m Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 17:55:58 [35mSTATE:[39m Build for: nodeGPU type: node: {"imports":39,"importBytes":446076,"outputBytes":398051,"outputFiles":"dist/human.node-gpu.js"}
|
||||
2021-05-30 17:55:58 [35mSTATE:[39m Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 17:55:58 [35mSTATE:[39m Build for: nodeWASM type: node: {"imports":39,"importBytes":446143,"outputBytes":398123,"outputFiles":"dist/human.node-wasm.js"}
|
||||
2021-05-30 17:55:58 [35mSTATE:[39m Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 17:55:58 [35mSTATE:[39m Build for: browserNoBundle type: esm: {"imports":39,"importBytes":446170,"outputBytes":243299,"outputFiles":"dist/human.esm-nobundle.js"}
|
||||
2021-05-30 17:55:59 [35mSTATE:[39m Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111418,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-05-30 17:55:59 [35mSTATE:[39m Build for: browserBundle type: iife: {"imports":39,"importBytes":1556194,"outputBytes":1351116,"outputFiles":"dist/human.js"}
|
||||
2021-05-30 17:56:00 [35mSTATE:[39m Build for: browserBundle type: esm: {"imports":39,"importBytes":1556194,"outputBytes":1351108,"outputFiles":"dist/human.esm.js"}
|
||||
2021-05-30 17:56:00 [36mINFO: [39m Generate types: ["src/human.ts"]
|
||||
2021-05-30 17:56:05 [36mINFO: [39m Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
|
||||
2021-05-30 17:56:05 [36mINFO: [39m Generate TypeDocs: ["src/human.ts"]
|
||||
|
|
|
@ -71,6 +71,8 @@ export const options: DrawOptions = {
|
|||
|
||||
let bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
|
||||
|
||||
const rad2deg = (theta) => Math.round((theta * 180) / Math.PI);
|
||||
|
||||
function point(ctx, x, y, z = 0, localOptions) {
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
|
||||
ctx.beginPath();
|
||||
|
@ -186,7 +188,10 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
|
|||
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
|
||||
labels.push(emotion.join(' '));
|
||||
}
|
||||
if (f.rotation && f.rotation.angle && f.rotation.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
|
||||
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
|
||||
if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`);
|
||||
if (f.rotation.gaze.angle) labels.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}°`);
|
||||
}
|
||||
if (labels.length === 0) labels.push('face');
|
||||
ctx.fillStyle = localOptions.color;
|
||||
for (let i = labels.length - 1; i >= 0; i--) {
|
||||
|
@ -553,7 +558,18 @@ function calcBuffered(newResult, localOptions) {
|
|||
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
|
||||
const boxRaw = newResult.face[i].boxRaw // update boxRaw
|
||||
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
|
||||
bufferedResult.face[i] = { ...newResult.face[i], box, boxRaw }; // shallow clone plus updated values
|
||||
const matrix = newResult.face[i].rotation.matrix;
|
||||
const angle = {
|
||||
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
|
||||
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
|
||||
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor,
|
||||
};
|
||||
const gaze = {
|
||||
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
|
||||
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor,
|
||||
};
|
||||
const rotation = { angle, matrix, gaze };
|
||||
bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values
|
||||
}
|
||||
|
||||
// interpolate person results
|
||||
|
|
|
@ -15,8 +15,8 @@ const rad2deg = (theta) => (theta * 180) / Math.PI;
|
|||
const calculateGaze = (mesh): { angle: number, strength: number } => {
|
||||
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points
|
||||
|
||||
const offsetIris = [0, 0]; // tbd: iris center may not align with average of eye extremes
|
||||
const eyeRatio = 5; // factor to normalize changes x vs y
|
||||
const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes
|
||||
const eyeRatio = 1; // factor to normalize changes x vs y
|
||||
|
||||
const left = mesh[33][2] > mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis
|
||||
const irisCenter = left ? mesh[473] : mesh[468];
|
||||
|
|
|
@ -41,6 +41,7 @@ export function process(input, config): { tensor: Tensor | null, canvas: Offscre
|
|||
// check if resizing will be needed
|
||||
const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0));
|
||||
const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0));
|
||||
if (!originalWidth || !originalHeight) return { tensor: null, canvas: inCanvas }; // video may become temporarily unavailable due to onresize
|
||||
let targetWidth = originalWidth;
|
||||
let targetHeight = originalHeight;
|
||||
if (targetWidth > maxSize) {
|
||||
|
|
Loading…
Reference in New Issue