implemented service worker

CHANGELOG.md

@@ -11,6 +11,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ### **HEAD -> main** 2021/05/30 mandic00@live.com
 
+- quantized centernet
+- release candidate
+- added usage restrictions
 - quantize handdetect model
 - added experimental movenet-lightning and removed blazepose from default dist
 - added experimental face.rotation.gaze

README.md

@@ -9,7 +9,7 @@
 
 **AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition,**  
 **Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis,**  
-**Age & Gender & Emotion Prediction, Gesture Recognition**  
+**Age & Gender & Emotion Prediction, Gaze Tracking, Gesture Recognition**  
 
 <br>
 

TODO.md

@@ -17,7 +17,6 @@ N/A
 
 ## In Progress
 
-- Face rotation interpolation
 - Object detection interpolation
 
 ## Issues

demo/index-pwa.js

@@ -0,0 +1,138 @@
+/**
+ * PWA Service Worker for Human main demo
+ */
+
+/// <reference lib="webworker" />
+
+// // @ts-nocheck Linting of ServiceWorker is not supported for JS files
+
+const skipCaching = false;
+
+const cacheName = 'Human';
+const cacheFiles = ['/favicon.ico', 'manifest.webmanifest']; // assets and models are cached on first access
+
+let cacheModels = true; // *.bin; *.json
+let cacheWASM = true; // *.wasm
+let cacheOther = false; // *
+
+let listening = false;
+const stats = { hit: 0, miss: 0 };
+
+const log = (...msg) => {
+  const dt = new Date();
+  const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`;
+  // eslint-disable-next-line no-console
+  console.log(ts, 'pwa', ...msg);
+};
+
+async function updateCached(req) {
+  fetch(req)
+    .then((update) => {
+      // update cache if request is ok
+      if (update.ok) {
+        caches
+          .open(cacheName)
+          .then((cache) => cache.put(req, update))
+          .catch((err) => log('cache update error', err));
+      }
+      return true;
+    })
+    .catch((err) => {
+      log('fetch error', err);
+      return false;
+    });
+}
+
+async function getCached(evt) {
+  // just fetch
+  if (skipCaching) return fetch(evt.request);
+
+  // get from cache or fetch if not in cache
+  let found = await caches.match(evt.request);
+  if (found && found.ok) {
+    stats.hit += 1;
+  } else {
+    stats.miss += 1;
+    found = await fetch(evt.request);
+  }
+
+  // if still don't have it, return offline page
+  if (!found || !found.ok) {
+    found = await caches.match('offline.html');
+  }
+
+  // update cache in the background
+  if (found && found.type === 'basic' && found.ok) {
+    const uri = new URL(evt.request.url);
+    if (uri.pathname.endsWith('.bin') || uri.pathname.endsWith('.json')) {
+      if (cacheModels) updateCached(evt.request);
+    } else if (uri.pathname.endsWith('.wasm')) {
+      if (cacheWASM) updateCached(evt.request);
+    } else if (cacheOther) {
+      updateCached(evt.request);
+    }
+  }
+
+  return found;
+}
+
+function cacheInit() {
+  // eslint-disable-next-line promise/catch-or-return
+  caches.open(cacheName)
+    // eslint-disable-next-line promise/no-nesting
+    .then((cache) => cache.addAll(cacheFiles)
+      .then(
+        () => log('cache refresh:', cacheFiles.length, 'files'),
+        (err) => log('cache error', err),
+      ));
+}
+
+if (!listening) {
+  // get messages from main app to update configuration
+  self.addEventListener('message', (evt) => {
+    log('event message:', evt.data);
+    switch (evt.data.key) {
+      case 'cacheModels': cacheModels = evt.data.val; break;
+      case 'cacheWASM': cacheWASM = evt.data.val; break;
+      case 'cacheOther': cacheOther = evt.data.val; break;
+      default:
+    }
+  });
+
+  self.addEventListener('install', (evt) => {
+    log('install');
+    // @ts-ignore scope for self is ServiceWorkerGlobalScope not Window
+    self.skipWaiting();
+    evt.waitUntil(cacheInit);
+  });
+
+  self.addEventListener('activate', (evt) => {
+    log('activate');
+    // @ts-ignore scope for self is ServiceWorkerGlobalScope not Window
+    evt.waitUntil(self.clients.claim());
+  });
+
+  self.addEventListener('fetch', (evt) => {
+    const uri = new URL(evt.request.url);
+    // if (uri.pathname === '/') { log('cache skip /', evt.request); return; } // skip root access requests
+    if (evt.request.cache === 'only-if-cached' && evt.request.mode !== 'same-origin') return; // required due to chrome bug
+    if (uri.origin !== location.origin) return; // skip non-local requests
+    if (evt.request.method !== 'GET') return; // only cache get requests
+    if (evt.request.url.includes('/api/')) return; // don't cache api requests, failures are handled at the time of call
+
+    const response = getCached(evt);
+    if (response) evt.respondWith(response);
+    else log('fetch response missing');
+  });
+
+  // only trigger controllerchange once
+  let refreshed = false;
+  self.addEventListener('controllerchange', (evt) => {
+    log(`PWA: ${evt.type}`);
+    if (refreshed) return;
+    refreshed = true;
+    location.reload();
+  });
+
+  listening = true;
+}

demo/index.js

@@ -36,24 +36,25 @@ const userConfig = {
     enabled: false,
     flip: false,
   },
-  face: { enabled: false,
+  face: { enabled: true,
     detector: { return: true },
     mesh: { enabled: true },
     iris: { enabled: true },
-    description: { enabled: false },
+    description: { enabled: true },
-    emotion: { enabled: false },
+    emotion: { enabled: true },
   },
   hand: { enabled: false },
   // body: { enabled: true, modelPath: 'posenet.json' },
   // body: { enabled: true, modelPath: 'blazepose.json' },
   body: { enabled: false, modelPath: 'movenet-lightning.json' },
-  object: { enabled: true },
+  object: { enabled: false },
   gesture: { enabled: true },
 };
 
 const drawOptions = {
   bufferedOutput: true, // experimental feature that makes draw functions interpolate results between each detection for smoother movement
-  bufferedFactor: 3, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
+  bufferedFactor: 4, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
+  drawGaze: true,
 };
 
 // ui options
@@ -110,6 +111,15 @@ const ui = {
   ],
 };
 
+const pwa = {
+  enabled: true,
+  cacheName: 'Human',
+  scriptFile: 'index-pwa.js',
+  cacheModels: true,
+  cacheWASM: true,
+  cacheOther: false,
+};
+
 // global variables
 const menu = {};
 let worker;
@@ -660,6 +670,43 @@ async function drawWarmup(res) {
   await human.draw.all(canvas, res, drawOptions);
 }
 
+async function pwaRegister() {
+  if (!pwa.enabled) return;
+  if ('serviceWorker' in navigator) {
+    try {
+      let found;
+      const regs = await navigator.serviceWorker.getRegistrations();
+      for (const reg of regs) {
+        log('pwa found:', reg.scope);
+        if (reg.scope.startsWith(location.origin)) found = reg;
+      }
+      if (!found) {
+        const reg = await navigator.serviceWorker.register(pwa.scriptFile, { scope: '/' });
+        found = reg;
+        log('pwa registered:', reg.scope);
+      }
+    } catch (err) {
+      if (err.name === 'SecurityError') log('pwa: ssl certificate is untrusted');
+      else log('pwa error:', err);
+    }
+    if (navigator.serviceWorker.controller) {
+      // update pwa configuration as it doesn't have access to it
+      navigator.serviceWorker.controller.postMessage({ key: 'cacheModels', val: pwa.cacheModels });
+      navigator.serviceWorker.controller.postMessage({ key: 'cacheWASM', val: pwa.cacheWASM });
+      navigator.serviceWorker.controller.postMessage({ key: 'cacheOther', val: pwa.cacheOther });
+
+      log('pwa ctive:', navigator.serviceWorker.controller.scriptURL);
+      const cache = await caches.open(pwa.cacheName);
+      if (cache) {
+        const content = await cache.matchAll();
+        log('pwa cache:', content.length, 'files');
+      }
+    }
+  } else {
+    log('pwa inactive');
+  }
+}
+
 async function main() {
   window.addEventListener('unhandledrejection', (evt) => {
     // eslint-disable-next-line no-console
@@ -679,6 +726,9 @@ async function main() {
     log('workers are disabled due to missing browser functionality');
   }
 
+  // register PWA ServiceWorker
+  await pwaRegister();
+
   // parse url search params
   const params = new URLSearchParams(location.search);
   log('url options:', params.toString());
@@ -735,10 +785,16 @@ async function main() {
   for (const m of Object.values(menu)) m.hide();
 
   if (params.has('image')) {
-    const image = JSON.parse(params.get('image'));
+    try {
-    log('overriding image:', image);
+      const image = JSON.parse(params.get('image'));
-    ui.samples = [image];
+      log('overriding image:', image);
-    await detectSampleImages();
+      ui.samples = [image];
+    } catch {
+      status('cannot parse input image');
+      log('cannot parse input image', params.get('image'));
+      ui.samples = [];
+    }
+    if (ui.samples.length > 0) await detectSampleImages();
   }
 
   if (params.has('images')) {

demo/offline.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta http-equiv="content-type" content="text/html; charset=utf-8">
+    <title>Human: Offline</title>
+    <meta name="viewport" content="width=device-width, shrink-to-fit=yes">
+    <meta name="mobile-web-app-capable" content="yes">
+    <meta name="application-name" content="Human">
+    <meta name="keywords" content="Human">
+    <meta name="description" content="Human; Author: Vladimir Mandic <mandic00@live.com>">
+    <meta name="msapplication-tooltip" content="Human; Author: Vladimir Mandic <mandic00@live.com>">
+    <link rel="manifest" href="manifest.webmanifest">
+    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
+    <link rel="icon" sizes="256x256" href="../assets/icon.png">
+    <link rel="apple-touch-icon" href="../assets/icon.png">
+    <link rel="apple-touch-startup-image" href="../assets/icon.png">
+    <style>
+      @font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../assets/lato-light.woff2') }
+      body { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; background: black; color: #ebebeb; }
+      h1 { font-size: 2rem; margin-top: 1.2rem; font-weight: bold; }
+      a { color: white; }
+      a:link { color: lightblue; text-decoration: none; }
+      a:hover { color: lightskyblue; text-decoration: none; }
+      .row { width: 90vw; margin: auto; margin-top: 100px; text-align: center; }
+    </style>
+</head>
+<body>
+  <div class="row text-center">
+    <h1>
+      <a href="/">Human: Offline</a><br>
+      <img alt="icon" src="../assets/icon.png">
+    </h1>
+  </div>
+</body>
+</html>

package.json

@@ -42,6 +42,7 @@
     "emotion-detection",
     "gender-prediction",
     "gesture-recognition",
+    "gaze-tracking",
     "age-gender",
     "faceapi",
     "face",

src/draw/draw.ts

@@ -71,6 +71,8 @@ export const options: DrawOptions = {
 
 let bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
 
+const rad2deg = (theta) => Math.round((theta * 180) / Math.PI);
+
 function point(ctx, x, y, z = 0, localOptions) {
   ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
   ctx.beginPath();
@@ -186,7 +188,10 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
       const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
       labels.push(emotion.join(' '));
     }
-    if (f.rotation && f.rotation.angle && f.rotation.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
+    if (f.rotation && f.rotation.angle && f.rotation.gaze) {
+      if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`);
+      if (f.rotation.gaze.angle) labels.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}°`);
+    }
     if (labels.length === 0) labels.push('face');
     ctx.fillStyle = localOptions.color;
     for (let i = labels.length - 1; i >= 0; i--) {
@@ -553,7 +558,18 @@ function calcBuffered(newResult, localOptions) {
       .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
     const boxRaw = newResult.face[i].boxRaw // update boxRaw
       .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
-    bufferedResult.face[i] = { ...newResult.face[i], box, boxRaw }; // shallow clone plus updated values
+    const matrix = newResult.face[i].rotation.matrix;
+    const angle = {
+      roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
+      yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
+      pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor,
+    };
+    const gaze = {
+      angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
+      strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor,
+    };
+    const rotation = { angle, matrix, gaze };
+    bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values
   }
 
   // interpolate person results

src/face.ts

@@ -15,8 +15,8 @@ const rad2deg = (theta) => (theta * 180) / Math.PI;
 const calculateGaze = (mesh): { angle: number, strength: number } => {
   const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points
 
-  const offsetIris = [0, 0]; // tbd: iris center may not align with average of eye extremes
+  const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes
-  const eyeRatio = 5; // factor to normalize changes x vs y
+  const eyeRatio = 1; // factor to normalize changes x vs y
 
   const left = mesh[33][2] > mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis
   const irisCenter = left ? mesh[473] : mesh[468];

src/image/image.ts

@@ -41,6 +41,7 @@ export function process(input, config): { tensor: Tensor | null, canvas: Offscre
     // check if resizing will be needed
     const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0));
     const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0));
+    if (!originalWidth || !originalHeight) return { tensor: null, canvas: inCanvas }; // video may become temporarily unavailable due to onresize
     let targetWidth = originalWidth;
     let targetHeight = originalHeight;
     if (targetWidth > maxSize) {