experimental webgpu support

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 pnpm-lock.yaml
+assets/tf*

CHANGELOG.md

@@ -11,6 +11,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ### **HEAD -> main** 2021/08/14 mandic00@live.com
 
+- add backend initialization checks
 - complete async work
 - list detect cameras
 - switch to async data reads

demo/helpers/menu.js

@@ -21,7 +21,7 @@ function createCSS() {
   if (CSScreated) return;
   const css = `
   :root { --rounded: 0.1rem; }
-  .menu { position: absolute; top: 0rem; right: 0; min-width: 180px; width: max-content; padding: 0.2rem 0.2rem 0 0.2rem; line-height: 1.8rem; z-index: 10; background: ${theme.background}; border: none }
+  .menu { position: absolute; top: 0rem; right: 0; min-width: 180px; width: max-content; padding: 0.2rem 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; background: ${theme.background}; border: none }
   .button { text-shadow: none; }
 
   .menu-container { display: block; max-height: 100vh; }
@@ -46,7 +46,7 @@ function createCSS() {
   .menu-button:hover { background: ${theme.buttonHover}; box-shadow: 4px 4px 4px 0 black; }
   .menu-button:focus { outline: none; }
 
-  .menu-checkbox { width: 2.6rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 0.5rem 0 0; position: relative; border-radius: var(--rounded); }
+  .menu-checkbox { width: 2.6rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 1.0rem 0 0; position: relative; border-radius: var(--rounded); }
   .menu-checkbox:after { content: 'OFF'; color: ${theme.checkboxOff}; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
   .menu-checkbox:before { content: 'ON'; color: ${theme.checkboxOn}; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
   .menu-checkbox-label { width: 1.3rem; height: 1rem; cursor: pointer; position: absolute; top: 0; left: 0rem; z-index: 1; background: ${theme.checkboxOff};
@@ -55,7 +55,7 @@ function createCSS() {
   input[type=checkbox] { visibility: hidden; }
   input[type=checkbox]:checked + label { left: 1.4rem; background: ${theme.checkboxOn}; }
 
-  .menu-range { margin: 0.2rem 0.5rem 0 0; width: 3.5rem; background: transparent; color: ${theme.rangeBackground}; }
+  .menu-range { margin: 0.2rem 1.0rem 0 0; width: 5rem; background: transparent; color: ${theme.rangeBackground}; }
   .menu-range:before { color: ${theme.rangeLabel}; margin: 0 0.4rem 0 0; font-weight: 800; font-size: 0.6rem; position: relative; top: 0.3rem; content: attr(value); }
 
   input[type=range] { -webkit-appearance: none; }

demo/index.js

@@ -199,6 +199,13 @@ async function calcSimmilarity(result) {
   document.getElementById('similarity').innerText = `similarity: ${Math.trunc(1000 * similarity) / 10}%`;
 }
 
+const isLive = (input) => {
+  const videoLive = input.readyState > 2;
+  const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live';
+  const live = (videoLive || cameraLive) && (!input.paused);
+  return live;
+};
+
 // draws processed results and starts processing of a next frame
 let lastDraw = performance.now();
 async function drawResults(input) {
@@ -271,11 +278,17 @@ async function drawResults(input) {
   ui.lastFrame = performance.now();
   // if buffered, immediate loop but limit frame rate although it's going to run slower as JS is singlethreaded
   if (ui.buffered) {
-    ui.drawThread = requestAnimationFrame(() => drawResults(input));
+    if (isLive(input)) {
+      ui.drawThread = requestAnimationFrame(() => drawResults(input));
+    } else {
+      cancelAnimationFrame(ui.drawThread);
+      ui.drawThread = null;
+    }
   } else {
     if (ui.drawThread) {
       log('stopping buffered refresh');
       cancelAnimationFrame(ui.drawThread);
+      ui.drawThread = null;
     }
   }
 }
@@ -326,7 +339,7 @@ async function setupCamera() {
   };
   // enumerate devices for diag purposes
   if (initialCameraAccess) {
-    navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated devices:', devices));
+    navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated input devices:', devices));
     log('camera constraints', constraints);
   }
   try {
@@ -365,7 +378,6 @@ async function setupCamera() {
       // eslint-disable-next-line no-use-before-define
       if (live && !ui.detectThread) runHumanDetect(video, canvas);
       ui.busy = false;
-      status();
       resolve();
     };
   });
@@ -402,6 +414,7 @@ function webWorker(input, image, canvas, timestamp) {
     worker = new Worker(ui.worker);
     // after receiving message from webworker, parse&draw results and send new frame for processing
     worker.addEventListener('message', (msg) => {
+      status();
       if (msg.data.result.performance && msg.data.result.performance.total) ui.detectFPS.push(1000 / msg.data.result.performance.total);
       if (ui.detectFPS.length > ui.maxFPSframes) ui.detectFPS.shift();
       if (ui.bench) {
@@ -421,14 +434,8 @@ function webWorker(input, image, canvas, timestamp) {
       }
 
       ui.framesDetect++;
-      if (!ui.drawThread) {
+      if (!ui.drawThread) drawResults(input);
-        status();
+      if (isLive(input)) {
-        drawResults(input);
-      }
-      const videoLive = (input.readyState > 2) && (!input.paused);
-      const cameraLive = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live') && !input.paused;
-      const live = videoLive || cameraLive;
-      if (live) {
         // eslint-disable-next-line no-use-before-define
         ui.detectThread = requestAnimationFrame((now) => runHumanDetect(input, canvas, now));
       }
@@ -441,16 +448,13 @@ function webWorker(input, image, canvas, timestamp) {
 // main processing function when input is webcam, can use direct invocation or web worker
 function runHumanDetect(input, canvas, timestamp) {
   // if live video
-  const videoLive = input.readyState > 2;
+  if (!isLive(input)) {
-  const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live';
-  const live = (videoLive || cameraLive) && (!input.paused);
-  if (!live) {
     // stop ui refresh
     // if (ui.drawThread) cancelAnimationFrame(ui.drawThread);
     if (ui.detectThread) cancelAnimationFrame(ui.detectThread);
-    // if we want to continue and camera not ready, retry in 0.5sec, else just give up
     if (input.paused) log('video paused');
-    else if (cameraLive && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500);
+    // if we want to continue and camera not ready, retry in 0.5sec, else just give up
+    // else if (cameraLive && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500);
     else log(`video not ready: track state: ${input.srcObject ? input.srcObject.getVideoTracks()[0].readyState : 'unknown'} stream state: ${input.readyState}`);
     log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw);
     log('memory', human.tf.engine().memory());
@@ -600,7 +604,6 @@ async function detectVideo() {
       document.getElementById('btnStartText').innerHTML = 'pause video';
       await video.play();
       runHumanDetect(video, canvas);
-      status();
     } else {
       status(cameraError);
     }
@@ -721,6 +724,8 @@ function setupMenu() {
     compare.original = null;
   });
 
+  for (const m of Object.values(menu)) m.hide();
+
   document.getElementById('btnDisplay').addEventListener('click', (evt) => menu.display.toggle(evt));
   document.getElementById('btnImage').addEventListener('click', (evt) => menu.image.toggle(evt));
   document.getElementById('btnProcess').addEventListener('click', (evt) => menu.process.toggle(evt));
@@ -974,7 +979,6 @@ async function main() {
   status('human: ready');
   document.getElementById('loader').style.display = 'none';
   document.getElementById('play').style.display = 'block';
-  for (const m of Object.values(menu)) m.hide();
 
   // init drag & drop
   await dragAndDrop();

src/draw/draw.ts

@@ -511,16 +511,17 @@ export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasE
 export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptions?: DrawOptions) {
   const timestamp = now();
   const localOptions = mergeDeep(options, drawOptions);
-  if (!result || !inCanvas) return;
+  if (!result || !inCanvas) return null;
-  if (!(inCanvas instanceof HTMLCanvasElement)) return;
+  if (!(inCanvas instanceof HTMLCanvasElement)) return null;
-
-  face(inCanvas, result.face, localOptions);
-  body(inCanvas, result.body, localOptions);
-  hand(inCanvas, result.hand, localOptions);
-  object(inCanvas, result.object, localOptions);
-  // person(inCanvas, result.persons, localOptions);
-  gesture(inCanvas, result.gesture, localOptions); // gestures do not have buffering
 
+  const promise = Promise.all([
+    face(inCanvas, result.face, localOptions),
+    body(inCanvas, result.body, localOptions),
+    hand(inCanvas, result.hand, localOptions),
+    object(inCanvas, result.object, localOptions),
+    // person(inCanvas, result.persons, localOptions);
+    gesture(inCanvas, result.gesture, localOptions), // gestures do not have buffering
+  ]);
   /*
   if (!bufferedResult) bufferedResult = result; // first pass
   else if (localOptions.bufferedOutput) calcBuffered(result); // do results interpolation
@@ -535,4 +536,5 @@ export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptio
   // await Promise.all(promises);
   */
   result.performance.draw = Math.trunc(now() - timestamp);
+  return promise;
 }