mirror of https://github.com/vladmandic/human
implemented service worker
Vladimir Mandic
parent
168fd473c6
commit
168ad14fda
|
|
@ -11,6 +11,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
### **HEAD -> main** 2021/05/30 mandic00@live.com
|
||||
|
||||
- quantized centernet
|
||||
- release candidate
|
||||
- added usage restrictions
|
||||
- quantize handdetect model
|
||||
- added experimental movenet-lightning and removed blazepose from default dist
|
||||
- added experimental face.rotation.gaze
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
**AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition,**
|
||||
**Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis,**
|
||||
**Age & Gender & Emotion Prediction, Gesture Recognition**
|
||||
**Age & Gender & Emotion Prediction, Gaze Tracking, Gesture Recognition**
|
||||
|
||||
<br>
|
||||
|
||||
|
|
|
|||
1
TODO.md
1
TODO.md
|
|
@ -17,7 +17,6 @@ N/A
|
|||
|
||||
## In Progress
|
||||
|
||||
- Face rotation interpolation
|
||||
- Object detection interpolation
|
||||
|
||||
## Issues
|
||||
|
|
|
|||
|
|
@ -0,0 +1,138 @@
|
|||
/**
|
||||
* PWA Service Worker for Human main demo
|
||||
*/
|
||||
|
||||
/// <reference lib="webworker" />
|
||||
|
||||
// // @ts-nocheck Linting of ServiceWorker is not supported for JS files
|
||||
|
||||
const skipCaching = false;
|
||||
|
||||
const cacheName = 'Human';
|
||||
const cacheFiles = ['/favicon.ico', 'manifest.webmanifest']; // assets and models are cached on first access
|
||||
|
||||
let cacheModels = true; // *.bin; *.json
|
||||
let cacheWASM = true; // *.wasm
|
||||
let cacheOther = false; // *
|
||||
|
||||
let listening = false;
|
||||
const stats = { hit: 0, miss: 0 };
|
||||
|
||||
const log = (...msg) => {
|
||||
const dt = new Date();
|
||||
const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`;
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(ts, 'pwa', ...msg);
|
||||
};
|
||||
|
||||
async function updateCached(req) {
|
||||
fetch(req)
|
||||
.then((update) => {
|
||||
// update cache if request is ok
|
||||
if (update.ok) {
|
||||
caches
|
||||
.open(cacheName)
|
||||
.then((cache) => cache.put(req, update))
|
||||
.catch((err) => log('cache update error', err));
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.catch((err) => {
|
||||
log('fetch error', err);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
async function getCached(evt) {
|
||||
// just fetch
|
||||
if (skipCaching) return fetch(evt.request);
|
||||
|
||||
// get from cache or fetch if not in cache
|
||||
let found = await caches.match(evt.request);
|
||||
if (found && found.ok) {
|
||||
stats.hit += 1;
|
||||
} else {
|
||||
stats.miss += 1;
|
||||
found = await fetch(evt.request);
|
||||
}
|
||||
|
||||
// if still don't have it, return offline page
|
||||
if (!found || !found.ok) {
|
||||
found = await caches.match('offline.html');
|
||||
}
|
||||
|
||||
// update cache in the background
|
||||
if (found && found.type === 'basic' && found.ok) {
|
||||
const uri = new URL(evt.request.url);
|
||||
if (uri.pathname.endsWith('.bin') || uri.pathname.endsWith('.json')) {
|
||||
if (cacheModels) updateCached(evt.request);
|
||||
} else if (uri.pathname.endsWith('.wasm')) {
|
||||
if (cacheWASM) updateCached(evt.request);
|
||||
} else if (cacheOther) {
|
||||
updateCached(evt.request);
|
||||
}
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
function cacheInit() {
|
||||
// eslint-disable-next-line promise/catch-or-return
|
||||
caches.open(cacheName)
|
||||
// eslint-disable-next-line promise/no-nesting
|
||||
.then((cache) => cache.addAll(cacheFiles)
|
||||
.then(
|
||||
() => log('cache refresh:', cacheFiles.length, 'files'),
|
||||
(err) => log('cache error', err),
|
||||
));
|
||||
}
|
||||
|
||||
if (!listening) {
|
||||
// get messages from main app to update configuration
|
||||
self.addEventListener('message', (evt) => {
|
||||
log('event message:', evt.data);
|
||||
switch (evt.data.key) {
|
||||
case 'cacheModels': cacheModels = evt.data.val; break;
|
||||
case 'cacheWASM': cacheWASM = evt.data.val; break;
|
||||
case 'cacheOther': cacheOther = evt.data.val; break;
|
||||
default:
|
||||
}
|
||||
});
|
||||
|
||||
self.addEventListener('install', (evt) => {
|
||||
log('install');
|
||||
// @ts-ignore scope for self is ServiceWorkerGlobalScope not Window
|
||||
self.skipWaiting();
|
||||
evt.waitUntil(cacheInit);
|
||||
});
|
||||
|
||||
self.addEventListener('activate', (evt) => {
|
||||
log('activate');
|
||||
// @ts-ignore scope for self is ServiceWorkerGlobalScope not Window
|
||||
evt.waitUntil(self.clients.claim());
|
||||
});
|
||||
|
||||
self.addEventListener('fetch', (evt) => {
|
||||
const uri = new URL(evt.request.url);
|
||||
// if (uri.pathname === '/') { log('cache skip /', evt.request); return; } // skip root access requests
|
||||
if (evt.request.cache === 'only-if-cached' && evt.request.mode !== 'same-origin') return; // required due to chrome bug
|
||||
if (uri.origin !== location.origin) return; // skip non-local requests
|
||||
if (evt.request.method !== 'GET') return; // only cache get requests
|
||||
if (evt.request.url.includes('/api/')) return; // don't cache api requests, failures are handled at the time of call
|
||||
|
||||
const response = getCached(evt);
|
||||
if (response) evt.respondWith(response);
|
||||
else log('fetch response missing');
|
||||
});
|
||||
|
||||
// only trigger controllerchange once
|
||||
let refreshed = false;
|
||||
self.addEventListener('controllerchange', (evt) => {
|
||||
log(`PWA: ${evt.type}`);
|
||||
if (refreshed) return;
|
||||
refreshed = true;
|
||||
location.reload();
|
||||
});
|
||||
|
||||
listening = true;
|
||||
}
|
||||
|
|
@ -36,24 +36,25 @@ const userConfig = {
|
|||
enabled: false,
|
||||
flip: false,
|
||||
},
|
||||
face: { enabled: false,
|
||||
face: { enabled: true,
|
||||
detector: { return: true },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
description: { enabled: false },
|
||||
emotion: { enabled: false },
|
||||
description: { enabled: true },
|
||||
emotion: { enabled: true },
|
||||
},
|
||||
hand: { enabled: false },
|
||||
// body: { enabled: true, modelPath: 'posenet.json' },
|
||||
// body: { enabled: true, modelPath: 'blazepose.json' },
|
||||
body: { enabled: false, modelPath: 'movenet-lightning.json' },
|
||||
object: { enabled: true },
|
||||
object: { enabled: false },
|
||||
gesture: { enabled: true },
|
||||
};
|
||||
|
||||
const drawOptions = {
|
||||
bufferedOutput: true, // experimental feature that makes draw functions interpolate results between each detection for smoother movement
|
||||
bufferedFactor: 3, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
|
||||
bufferedFactor: 4, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
|
||||
drawGaze: true,
|
||||
};
|
||||
|
||||
// ui options
|
||||
|
|
@ -110,6 +111,15 @@ const ui = {
|
|||
],
|
||||
};
|
||||
|
||||
const pwa = {
|
||||
enabled: true,
|
||||
cacheName: 'Human',
|
||||
scriptFile: 'index-pwa.js',
|
||||
cacheModels: true,
|
||||
cacheWASM: true,
|
||||
cacheOther: false,
|
||||
};
|
||||
|
||||
// global variables
|
||||
const menu = {};
|
||||
let worker;
|
||||
|
|
@ -660,6 +670,43 @@ async function drawWarmup(res) {
|
|||
await human.draw.all(canvas, res, drawOptions);
|
||||
}
|
||||
|
||||
async function pwaRegister() {
|
||||
if (!pwa.enabled) return;
|
||||
if ('serviceWorker' in navigator) {
|
||||
try {
|
||||
let found;
|
||||
const regs = await navigator.serviceWorker.getRegistrations();
|
||||
for (const reg of regs) {
|
||||
log('pwa found:', reg.scope);
|
||||
if (reg.scope.startsWith(location.origin)) found = reg;
|
||||
}
|
||||
if (!found) {
|
||||
const reg = await navigator.serviceWorker.register(pwa.scriptFile, { scope: '/' });
|
||||
found = reg;
|
||||
log('pwa registered:', reg.scope);
|
||||
}
|
||||
} catch (err) {
|
||||
if (err.name === 'SecurityError') log('pwa: ssl certificate is untrusted');
|
||||
else log('pwa error:', err);
|
||||
}
|
||||
if (navigator.serviceWorker.controller) {
|
||||
// update pwa configuration as it doesn't have access to it
|
||||
navigator.serviceWorker.controller.postMessage({ key: 'cacheModels', val: pwa.cacheModels });
|
||||
navigator.serviceWorker.controller.postMessage({ key: 'cacheWASM', val: pwa.cacheWASM });
|
||||
navigator.serviceWorker.controller.postMessage({ key: 'cacheOther', val: pwa.cacheOther });
|
||||
|
||||
log('pwa ctive:', navigator.serviceWorker.controller.scriptURL);
|
||||
const cache = await caches.open(pwa.cacheName);
|
||||
if (cache) {
|
||||
const content = await cache.matchAll();
|
||||
log('pwa cache:', content.length, 'files');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log('pwa inactive');
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
window.addEventListener('unhandledrejection', (evt) => {
|
||||
// eslint-disable-next-line no-console
|
||||
|
|
@ -679,6 +726,9 @@ async function main() {
|
|||
log('workers are disabled due to missing browser functionality');
|
||||
}
|
||||
|
||||
// register PWA ServiceWorker
|
||||
await pwaRegister();
|
||||
|
||||
// parse url search params
|
||||
const params = new URLSearchParams(location.search);
|
||||
log('url options:', params.toString());
|
||||
|
|
@ -735,10 +785,16 @@ async function main() {
|
|||
for (const m of Object.values(menu)) m.hide();
|
||||
|
||||
if (params.has('image')) {
|
||||
const image = JSON.parse(params.get('image'));
|
||||
log('overriding image:', image);
|
||||
ui.samples = [image];
|
||||
await detectSampleImages();
|
||||
try {
|
||||
const image = JSON.parse(params.get('image'));
|
||||
log('overriding image:', image);
|
||||
ui.samples = [image];
|
||||
} catch {
|
||||
status('cannot parse input image');
|
||||
log('cannot parse input image', params.get('image'));
|
||||
ui.samples = [];
|
||||
}
|
||||
if (ui.samples.length > 0) await detectSampleImages();
|
||||
}
|
||||
|
||||
if (params.has('images')) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="content-type" content="text/html; charset=utf-8">
|
||||
<title>Human: Offline</title>
|
||||
<meta name="viewport" content="width=device-width, shrink-to-fit=yes">
|
||||
<meta name="mobile-web-app-capable" content="yes">
|
||||
<meta name="application-name" content="Human">
|
||||
<meta name="keywords" content="Human">
|
||||
<meta name="description" content="Human; Author: Vladimir Mandic <mandic00@live.com>">
|
||||
<meta name="msapplication-tooltip" content="Human; Author: Vladimir Mandic <mandic00@live.com>">
|
||||
<link rel="manifest" href="manifest.webmanifest">
|
||||
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
|
||||
<link rel="icon" sizes="256x256" href="../assets/icon.png">
|
||||
<link rel="apple-touch-icon" href="../assets/icon.png">
|
||||
<link rel="apple-touch-startup-image" href="../assets/icon.png">
|
||||
<style>
|
||||
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../assets/lato-light.woff2') }
|
||||
body { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; background: black; color: #ebebeb; }
|
||||
h1 { font-size: 2rem; margin-top: 1.2rem; font-weight: bold; }
|
||||
a { color: white; }
|
||||
a:link { color: lightblue; text-decoration: none; }
|
||||
a:hover { color: lightskyblue; text-decoration: none; }
|
||||
.row { width: 90vw; margin: auto; margin-top: 100px; text-align: center; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="row text-center">
|
||||
<h1>
|
||||
<a href="/">Human: Offline</a><br>
|
||||
<img alt="icon" src="../assets/icon.png">
|
||||
</h1>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -42,6 +42,7 @@
|
|||
"emotion-detection",
|
||||
"gender-prediction",
|
||||
"gesture-recognition",
|
||||
"gaze-tracking",
|
||||
"age-gender",
|
||||
"faceapi",
|
||||
"face",
|
||||
|
|
|
|||
|
|
@ -71,6 +71,8 @@ export const options: DrawOptions = {
|
|||
|
||||
let bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
|
||||
|
||||
const rad2deg = (theta) => Math.round((theta * 180) / Math.PI);
|
||||
|
||||
function point(ctx, x, y, z = 0, localOptions) {
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
|
||||
ctx.beginPath();
|
||||
|
|
@ -186,7 +188,10 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
|
|||
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
|
||||
labels.push(emotion.join(' '));
|
||||
}
|
||||
if (f.rotation && f.rotation.angle && f.rotation.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
|
||||
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
|
||||
if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`);
|
||||
if (f.rotation.gaze.angle) labels.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}°`);
|
||||
}
|
||||
if (labels.length === 0) labels.push('face');
|
||||
ctx.fillStyle = localOptions.color;
|
||||
for (let i = labels.length - 1; i >= 0; i--) {
|
||||
|
|
@ -553,7 +558,18 @@ function calcBuffered(newResult, localOptions) {
|
|||
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
|
||||
const boxRaw = newResult.face[i].boxRaw // update boxRaw
|
||||
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
|
||||
bufferedResult.face[i] = { ...newResult.face[i], box, boxRaw }; // shallow clone plus updated values
|
||||
const matrix = newResult.face[i].rotation.matrix;
|
||||
const angle = {
|
||||
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
|
||||
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
|
||||
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor,
|
||||
};
|
||||
const gaze = {
|
||||
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
|
||||
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor,
|
||||
};
|
||||
const rotation = { angle, matrix, gaze };
|
||||
bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values
|
||||
}
|
||||
|
||||
// interpolate person results
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ const rad2deg = (theta) => (theta * 180) / Math.PI;
|
|||
const calculateGaze = (mesh): { angle: number, strength: number } => {
|
||||
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points
|
||||
|
||||
const offsetIris = [0, 0]; // tbd: iris center may not align with average of eye extremes
|
||||
const eyeRatio = 5; // factor to normalize changes x vs y
|
||||
const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes
|
||||
const eyeRatio = 1; // factor to normalize changes x vs y
|
||||
|
||||
const left = mesh[33][2] > mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis
|
||||
const irisCenter = left ? mesh[473] : mesh[468];
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ export function process(input, config): { tensor: Tensor | null, canvas: Offscre
|
|||
// check if resizing will be needed
|
||||
const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0));
|
||||
const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0));
|
||||
if (!originalWidth || !originalHeight) return { tensor: null, canvas: inCanvas }; // video may become temporarily unavailable due to onresize
|
||||
let targetWidth = originalWidth;
|
||||
let targetHeight = originalHeight;
|
||||
if (targetWidth > maxSize) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue