From 9aaa83539518bdef3e6648e8c6bc248d7afb156e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 30 May 2021 17:56:40 -0400 Subject: [PATCH] implemented service worker --- CHANGELOG.md | 3 + README.md | 2 +- TODO.md | 1 - demo/index-pwa.js | 138 +++++++++++++++++++++++++++++++++++++++++++++ demo/index.js | 74 +++++++++++++++++++++--- demo/offline.html | 35 ++++++++++++ package.json | 1 + src/draw/draw.ts | 20 ++++++- src/face.ts | 4 +- src/image/image.ts | 1 + 10 files changed, 264 insertions(+), 15 deletions(-) create mode 100644 demo/index-pwa.js create mode 100644 demo/offline.html diff --git a/CHANGELOG.md b/CHANGELOG.md index c725042a..15c851c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ Repository: **** ### **HEAD -> main** 2021/05/30 mandic00@live.com +- quantized centernet +- release candidate +- added usage restrictions - quantize handdetect model - added experimental movenet-lightning and removed blazepose from default dist - added experimental face.rotation.gaze diff --git a/README.md b/README.md index a369c4ac..d3a6d394 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ **AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition,** **Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis,** -**Age & Gender & Emotion Prediction, Gesture Recognition** +**Age & Gender & Emotion Prediction, Gaze Tracking, Gesture Recognition**
diff --git a/TODO.md b/TODO.md index 55c0aadf..38d9b189 100644 --- a/TODO.md +++ b/TODO.md @@ -17,7 +17,6 @@ N/A ## In Progress -- Face rotation interpolation - Object detection interpolation ## Issues diff --git a/demo/index-pwa.js b/demo/index-pwa.js new file mode 100644 index 00000000..a8961083 --- /dev/null +++ b/demo/index-pwa.js @@ -0,0 +1,138 @@ +/** + * PWA Service Worker for Human main demo + */ + +/// + +// // @ts-nocheck Linting of ServiceWorker is not supported for JS files + +const skipCaching = false; + +const cacheName = 'Human'; +const cacheFiles = ['/favicon.ico', 'manifest.webmanifest']; // assets and models are cached on first access + +let cacheModels = true; // *.bin; *.json +let cacheWASM = true; // *.wasm +let cacheOther = false; // * + +let listening = false; +const stats = { hit: 0, miss: 0 }; + +const log = (...msg) => { + const dt = new Date(); + const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`; + // eslint-disable-next-line no-console + console.log(ts, 'pwa', ...msg); +}; + +async function updateCached(req) { + fetch(req) + .then((update) => { + // update cache if request is ok + if (update.ok) { + caches + .open(cacheName) + .then((cache) => cache.put(req, update)) + .catch((err) => log('cache update error', err)); + } + return true; + }) + .catch((err) => { + log('fetch error', err); + return false; + }); +} + +async function getCached(evt) { + // just fetch + if (skipCaching) return fetch(evt.request); + + // get from cache or fetch if not in cache + let found = await caches.match(evt.request); + if (found && found.ok) { + stats.hit += 1; + } else { + stats.miss += 1; + found = await fetch(evt.request); + } + + // if still don't have it, return offline page + if (!found || !found.ok) { + found = await caches.match('offline.html'); + } + + // update cache in the background + if (found && found.type === 'basic' && found.ok) { + const uri = new URL(evt.request.url); + if (uri.pathname.endsWith('.bin') || uri.pathname.endsWith('.json')) { + if (cacheModels) updateCached(evt.request); + } else if (uri.pathname.endsWith('.wasm')) { + if (cacheWASM) updateCached(evt.request); + } else if (cacheOther) { + updateCached(evt.request); + } + } + + return found; +} + +function cacheInit() { + // eslint-disable-next-line promise/catch-or-return + caches.open(cacheName) + // eslint-disable-next-line promise/no-nesting + .then((cache) => cache.addAll(cacheFiles) + .then( + () => log('cache refresh:', cacheFiles.length, 'files'), + (err) => log('cache error', err), + )); +} + +if (!listening) { + // get messages from main app to update configuration + self.addEventListener('message', (evt) => { + log('event message:', evt.data); + switch (evt.data.key) { + case 'cacheModels': cacheModels = evt.data.val; break; + case 'cacheWASM': cacheWASM = evt.data.val; break; + case 'cacheOther': cacheOther = evt.data.val; break; + default: + } + }); + + self.addEventListener('install', (evt) => { + log('install'); + // @ts-ignore scope for self is ServiceWorkerGlobalScope not Window + self.skipWaiting(); + evt.waitUntil(cacheInit); + }); + + self.addEventListener('activate', (evt) => { + log('activate'); + // @ts-ignore scope for self is ServiceWorkerGlobalScope not Window + evt.waitUntil(self.clients.claim()); + }); + + self.addEventListener('fetch', (evt) => { + const uri = new URL(evt.request.url); + // if (uri.pathname === '/') { log('cache skip /', evt.request); return; } // skip root access requests + if (evt.request.cache === 'only-if-cached' && evt.request.mode !== 'same-origin') return; // required due to chrome bug + if (uri.origin !== location.origin) return; // skip non-local requests + if (evt.request.method !== 'GET') return; // only cache get requests + if (evt.request.url.includes('/api/')) return; // don't cache api requests, failures are handled at the time of call + + const response = getCached(evt); + if (response) evt.respondWith(response); + else log('fetch response missing'); + }); + + // only trigger controllerchange once + let refreshed = false; + self.addEventListener('controllerchange', (evt) => { + log(`PWA: ${evt.type}`); + if (refreshed) return; + refreshed = true; + location.reload(); + }); + + listening = true; +} diff --git a/demo/index.js b/demo/index.js index b62e7521..61774063 100644 --- a/demo/index.js +++ b/demo/index.js @@ -36,24 +36,25 @@ const userConfig = { enabled: false, flip: false, }, - face: { enabled: false, + face: { enabled: true, detector: { return: true }, mesh: { enabled: true }, iris: { enabled: true }, - description: { enabled: false }, - emotion: { enabled: false }, + description: { enabled: true }, + emotion: { enabled: true }, }, hand: { enabled: false }, // body: { enabled: true, modelPath: 'posenet.json' }, // body: { enabled: true, modelPath: 'blazepose.json' }, body: { enabled: false, modelPath: 'movenet-lightning.json' }, - object: { enabled: true }, + object: { enabled: false }, gesture: { enabled: true }, }; const drawOptions = { bufferedOutput: true, // experimental feature that makes draw functions interpolate results between each detection for smoother movement - bufferedFactor: 3, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc. + bufferedFactor: 4, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc. + drawGaze: true, }; // ui options @@ -110,6 +111,15 @@ const ui = { ], }; +const pwa = { + enabled: true, + cacheName: 'Human', + scriptFile: 'index-pwa.js', + cacheModels: true, + cacheWASM: true, + cacheOther: false, +}; + // global variables const menu = {}; let worker; @@ -660,6 +670,43 @@ async function drawWarmup(res) { await human.draw.all(canvas, res, drawOptions); } +async function pwaRegister() { + if (!pwa.enabled) return; + if ('serviceWorker' in navigator) { + try { + let found; + const regs = await navigator.serviceWorker.getRegistrations(); + for (const reg of regs) { + log('pwa found:', reg.scope); + if (reg.scope.startsWith(location.origin)) found = reg; + } + if (!found) { + const reg = await navigator.serviceWorker.register(pwa.scriptFile, { scope: '/' }); + found = reg; + log('pwa registered:', reg.scope); + } + } catch (err) { + if (err.name === 'SecurityError') log('pwa: ssl certificate is untrusted'); + else log('pwa error:', err); + } + if (navigator.serviceWorker.controller) { + // update pwa configuration as it doesn't have access to it + navigator.serviceWorker.controller.postMessage({ key: 'cacheModels', val: pwa.cacheModels }); + navigator.serviceWorker.controller.postMessage({ key: 'cacheWASM', val: pwa.cacheWASM }); + navigator.serviceWorker.controller.postMessage({ key: 'cacheOther', val: pwa.cacheOther }); + + log('pwa ctive:', navigator.serviceWorker.controller.scriptURL); + const cache = await caches.open(pwa.cacheName); + if (cache) { + const content = await cache.matchAll(); + log('pwa cache:', content.length, 'files'); + } + } + } else { + log('pwa inactive'); + } +} + async function main() { window.addEventListener('unhandledrejection', (evt) => { // eslint-disable-next-line no-console @@ -679,6 +726,9 @@ async function main() { log('workers are disabled due to missing browser functionality'); } + // register PWA ServiceWorker + await pwaRegister(); + // parse url search params const params = new URLSearchParams(location.search); log('url options:', params.toString()); @@ -735,10 +785,16 @@ async function main() { for (const m of Object.values(menu)) m.hide(); if (params.has('image')) { - const image = JSON.parse(params.get('image')); - log('overriding image:', image); - ui.samples = [image]; - await detectSampleImages(); + try { + const image = JSON.parse(params.get('image')); + log('overriding image:', image); + ui.samples = [image]; + } catch { + status('cannot parse input image'); + log('cannot parse input image', params.get('image')); + ui.samples = []; + } + if (ui.samples.length > 0) await detectSampleImages(); } if (params.has('images')) { diff --git a/demo/offline.html b/demo/offline.html new file mode 100644 index 00000000..b0fb540e --- /dev/null +++ b/demo/offline.html @@ -0,0 +1,35 @@ + + + + + Human: Offline + + + + + + + + + + + + + + +
+

+ Human: Offline
+ icon +

+
+ + diff --git a/package.json b/package.json index aae38dbe..4da3536f 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "emotion-detection", "gender-prediction", "gesture-recognition", + "gaze-tracking", "age-gender", "faceapi", "face", diff --git a/src/draw/draw.ts b/src/draw/draw.ts index 70d61da3..d2d6a78c 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -71,6 +71,8 @@ export const options: DrawOptions = { let bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 }; +const rad2deg = (theta) => Math.round((theta * 180) / Math.PI); + function point(ctx, x, y, z = 0, localOptions) { ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color; ctx.beginPath(); @@ -186,7 +188,10 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array, dra const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`); labels.push(emotion.join(' ')); } - if (f.rotation && f.rotation.angle && f.rotation.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`); + if (f.rotation && f.rotation.angle && f.rotation.gaze) { + if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`); + if (f.rotation.gaze.angle) labels.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}°`); + } if (labels.length === 0) labels.push('face'); ctx.fillStyle = localOptions.color; for (let i = labels.length - 1; i >= 0; i--) { @@ -553,7 +558,18 @@ function calcBuffered(newResult, localOptions) { .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor); const boxRaw = newResult.face[i].boxRaw // update boxRaw .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor); - bufferedResult.face[i] = { ...newResult.face[i], box, boxRaw }; // shallow clone plus updated values + const matrix = newResult.face[i].rotation.matrix; + const angle = { + roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor, + yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor, + pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor, + }; + const gaze = { + angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor, + strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor, + }; + const rotation = { angle, matrix, gaze }; + bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values } // interpolate person results diff --git a/src/face.ts b/src/face.ts index c544dd9e..16be6511 100644 --- a/src/face.ts +++ b/src/face.ts @@ -15,8 +15,8 @@ const rad2deg = (theta) => (theta * 180) / Math.PI; const calculateGaze = (mesh): { angle: number, strength: number } => { const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points - const offsetIris = [0, 0]; // tbd: iris center may not align with average of eye extremes - const eyeRatio = 5; // factor to normalize changes x vs y + const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes + const eyeRatio = 1; // factor to normalize changes x vs y const left = mesh[33][2] > mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis const irisCenter = left ? mesh[473] : mesh[468]; diff --git a/src/image/image.ts b/src/image/image.ts index 1991baab..2b446511 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -41,6 +41,7 @@ export function process(input, config): { tensor: Tensor | null, canvas: Offscre // check if resizing will be needed const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0)); const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0)); + if (!originalWidth || !originalHeight) return { tensor: null, canvas: inCanvas }; // video may become temporarily unavailable due to onresize let targetWidth = originalWidth; let targetHeight = originalHeight; if (targetWidth > maxSize) {