experimental webgpu support

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 pnpm-lock.yaml
+assets/tf*

CHANGELOG.md

@@ -11,6 +11,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ### **HEAD -> main** 2021/08/14 mandic00@live.com
 
+- add backend initialization checks
 - complete async work
 - list detect cameras
 - switch to async data reads

demo/helpers/menu.js

@@ -21,7 +21,7 @@ function createCSS() {
   if (CSScreated) return;
   const css = `
   :root { --rounded: 0.1rem; }
-  .menu { position: absolute; top: 0rem; right: 0; min-width: 180px; width: max-content; padding: 0.2rem 0.2rem 0 0.2rem; line-height: 1.8rem; z-index: 10; background: ${theme.background}; border: none }
+  .menu { position: absolute; top: 0rem; right: 0; min-width: 180px; width: max-content; padding: 0.2rem 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; background: ${theme.background}; border: none }
   .button { text-shadow: none; }
 
   .menu-container { display: block; max-height: 100vh; }
@@ -46,7 +46,7 @@ function createCSS() {
   .menu-button:hover { background: ${theme.buttonHover}; box-shadow: 4px 4px 4px 0 black; }
   .menu-button:focus { outline: none; }
 
-  .menu-checkbox { width: 2.6rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 0.5rem 0 0; position: relative; border-radius: var(--rounded); }
+  .menu-checkbox { width: 2.6rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 1.0rem 0 0; position: relative; border-radius: var(--rounded); }
   .menu-checkbox:after { content: 'OFF'; color: ${theme.checkboxOff}; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
   .menu-checkbox:before { content: 'ON'; color: ${theme.checkboxOn}; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
   .menu-checkbox-label { width: 1.3rem; height: 1rem; cursor: pointer; position: absolute; top: 0; left: 0rem; z-index: 1; background: ${theme.checkboxOff};
@@ -55,7 +55,7 @@ function createCSS() {
   input[type=checkbox] { visibility: hidden; }
   input[type=checkbox]:checked + label { left: 1.4rem; background: ${theme.checkboxOn}; }
 
-  .menu-range { margin: 0.2rem 0.5rem 0 0; width: 3.5rem; background: transparent; color: ${theme.rangeBackground}; }
+  .menu-range { margin: 0.2rem 1.0rem 0 0; width: 5rem; background: transparent; color: ${theme.rangeBackground}; }
   .menu-range:before { color: ${theme.rangeLabel}; margin: 0 0.4rem 0 0; font-weight: 800; font-size: 0.6rem; position: relative; top: 0.3rem; content: attr(value); }
 
   input[type=range] { -webkit-appearance: none; }

demo/index.js

@@ -199,6 +199,13 @@ async function calcSimmilarity(result) {
   document.getElementById('similarity').innerText = `similarity: ${Math.trunc(1000 * similarity) / 10}%`;
 }
 
+const isLive = (input) => {
+  const videoLive = input.readyState > 2;
+  const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live';
+  const live = (videoLive || cameraLive) && (!input.paused);
+  return live;
+};
+
 // draws processed results and starts processing of a next frame
 let lastDraw = performance.now();
 async function drawResults(input) {
@@ -271,11 +278,17 @@ async function drawResults(input) {
   ui.lastFrame = performance.now();
   // if buffered, immediate loop but limit frame rate although it's going to run slower as JS is singlethreaded
   if (ui.buffered) {
-    ui.drawThread = requestAnimationFrame(() => drawResults(input));
+    if (isLive(input)) {
+      ui.drawThread = requestAnimationFrame(() => drawResults(input));
+    } else {
+      cancelAnimationFrame(ui.drawThread);
+      ui.drawThread = null;
+    }
   } else {
     if (ui.drawThread) {
       log('stopping buffered refresh');
       cancelAnimationFrame(ui.drawThread);
+      ui.drawThread = null;
     }
   }
 }
@@ -326,7 +339,7 @@ async function setupCamera() {
   };
   // enumerate devices for diag purposes
   if (initialCameraAccess) {
-    navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated devices:', devices));
+    navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated input devices:', devices));
     log('camera constraints', constraints);
   }
   try {
@@ -365,7 +378,6 @@ async function setupCamera() {
       // eslint-disable-next-line no-use-before-define
       if (live && !ui.detectThread) runHumanDetect(video, canvas);
       ui.busy = false;
-      status();
       resolve();
     };
   });
@@ -402,6 +414,7 @@ function webWorker(input, image, canvas, timestamp) {
     worker = new Worker(ui.worker);
     // after receiving message from webworker, parse&draw results and send new frame for processing
     worker.addEventListener('message', (msg) => {
+      status();
       if (msg.data.result.performance && msg.data.result.performance.total) ui.detectFPS.push(1000 / msg.data.result.performance.total);
       if (ui.detectFPS.length > ui.maxFPSframes) ui.detectFPS.shift();
       if (ui.bench) {
@@ -421,14 +434,8 @@ function webWorker(input, image, canvas, timestamp) {
       }
 
       ui.framesDetect++;
-      if (!ui.drawThread) {
-        status();
-        drawResults(input);
-      }
-      const videoLive = (input.readyState > 2) && (!input.paused);
-      const cameraLive = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live') && !input.paused;
-      const live = videoLive || cameraLive;
-      if (live) {
+      if (!ui.drawThread) drawResults(input);
+      if (isLive(input)) {
         // eslint-disable-next-line no-use-before-define
         ui.detectThread = requestAnimationFrame((now) => runHumanDetect(input, canvas, now));
       }
@@ -441,16 +448,13 @@ function webWorker(input, image, canvas, timestamp) {
 // main processing function when input is webcam, can use direct invocation or web worker
 function runHumanDetect(input, canvas, timestamp) {
   // if live video
-  const videoLive = input.readyState > 2;
-  const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live';
-  const live = (videoLive || cameraLive) && (!input.paused);
-  if (!live) {
+  if (!isLive(input)) {
     // stop ui refresh
     // if (ui.drawThread) cancelAnimationFrame(ui.drawThread);
     if (ui.detectThread) cancelAnimationFrame(ui.detectThread);
-    // if we want to continue and camera not ready, retry in 0.5sec, else just give up
     if (input.paused) log('video paused');
-    else if (cameraLive && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500);
+    // if we want to continue and camera not ready, retry in 0.5sec, else just give up
+    // else if (cameraLive && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500);
     else log(`video not ready: track state: ${input.srcObject ? input.srcObject.getVideoTracks()[0].readyState : 'unknown'} stream state: ${input.readyState}`);
     log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw);
     log('memory', human.tf.engine().memory());
@@ -600,7 +604,6 @@ async function detectVideo() {
       document.getElementById('btnStartText').innerHTML = 'pause video';
       await video.play();
       runHumanDetect(video, canvas);
-      status();
     } else {
       status(cameraError);
     }
@@ -721,6 +724,8 @@ function setupMenu() {
     compare.original = null;
   });
 
+  for (const m of Object.values(menu)) m.hide();
+
   document.getElementById('btnDisplay').addEventListener('click', (evt) => menu.display.toggle(evt));
   document.getElementById('btnImage').addEventListener('click', (evt) => menu.image.toggle(evt));
   document.getElementById('btnProcess').addEventListener('click', (evt) => menu.process.toggle(evt));
@@ -974,7 +979,6 @@ async function main() {
   status('human: ready');
   document.getElementById('loader').style.display = 'none';
   document.getElementById('play').style.display = 'block';
-  for (const m of Object.values(menu)) m.hide();
 
   // init drag & drop
   await dragAndDrop();

demo/webgpu/index.js

@@ -7,7 +7,7 @@
 // @ts-nocheck // typescript checks disabled as this is pure javascript
 
 import '../../node_modules/@tensorflow/tfjs-core/dist/tf-core.es2017.js';
-import '../../node_modules/@tensorflow/tfjs-backend-webgpu/dist/tf-backend-webgpu.es2017.js';
+import '../../assets/tf-backend-webgpu.es2017.js';
 import Human from '../../dist/human.esm.js';
 
 let human;
@@ -16,33 +16,30 @@ let video;
 let result;
 
 const myConfig = {
-  backend: 'webgpu',
-  async: false,
+  backend: 'webgl',
+  async: true,
   warmup: 'none',
   modelBasePath: '../../models',
   cacheSensitivity: 0,
   filter: {
-    enabled: false,
+    enabled: true,
     flip: false,
   },
-  face: { enabled: false,
+  face: { enabled: true,
     detector: { return: false, rotation: false },
-    mesh: { enabled: false },
+    mesh: { enabled: true },
     iris: { enabled: false },
-    description: { enabled: false },
+    description: { enabled: true },
     emotion: { enabled: false },
   },
   object: { enabled: false },
-  gesture: { enabled: false },
-  hand: { enabled: false },
+  gesture: { enabled: true },
+  hand: { enabled: true, rotation: false },
   body: { enabled: true },
   segmentation: { enabled: false },
 };
 
-const time = {
-  detect: 0,
-  draw: 0,
-};
+let time = 0;
 
 function log(...msg) {
   const dt = new Date();
@@ -53,13 +50,17 @@ function log(...msg) {
 
 async function drawResults() {
   const interpolated = human.next(result);
+  const ctx = canvas.getContext('2d');
+  ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
   await human.draw.all(canvas, interpolated);
-  document.getElementById('log').innerText = `Human: version ${human.version} | FPS: ${1000 / time.detect} / ${1000 / time.draw}`;
+  document.getElementById('log').innerText = `Human: version ${human.version} | ${Math.trunc(time)} ms | FPS: ${Math.trunc(10000 / time) / 10}`;
   requestAnimationFrame(drawResults);
 }
 
 async function runDetection() {
+  const t0 = performance.now();
   result = await human.detect(video);
+  time = performance.now() - t0;
   requestAnimationFrame(runDetection);
 }
 
@@ -79,7 +80,7 @@ async function setupCamera() {
     },
   };
   // enumerate devices for diag purposes
-  navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated devices:', devices));
+  navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated input devices:', devices));
   log('camera constraints', constraints);
   try {
     stream = await navigator.mediaDevices.getUserMedia(constraints);

dist/human.node-gpu.js

@@ -10466,15 +10466,18 @@ async function all(inCanvas2, result, drawOptions) {
   const timestamp = now();
   const localOptions = mergeDeep(options, drawOptions);
   if (!result || !inCanvas2)
-    return;
+    return null;
   if (!(inCanvas2 instanceof HTMLCanvasElement))
-    return;
-  face2(inCanvas2, result.face, localOptions);
-  body2(inCanvas2, result.body, localOptions);
-  hand2(inCanvas2, result.hand, localOptions);
-  object(inCanvas2, result.object, localOptions);
-  gesture(inCanvas2, result.gesture, localOptions);
+    return null;
+  const promise = Promise.all([
+    face2(inCanvas2, result.face, localOptions),
+    body2(inCanvas2, result.body, localOptions),
+    hand2(inCanvas2, result.hand, localOptions),
+    object(inCanvas2, result.object, localOptions),
+    gesture(inCanvas2, result.gesture, localOptions)
+  ]);
   result.performance.draw = Math.trunc(now() - timestamp);
+  return promise;
 }
 
 // src/persons.ts

dist/human.node-wasm.js

@@ -10467,15 +10467,18 @@ async function all(inCanvas2, result, drawOptions) {
   const timestamp = now();
   const localOptions = mergeDeep(options, drawOptions);
   if (!result || !inCanvas2)
-    return;
+    return null;
   if (!(inCanvas2 instanceof HTMLCanvasElement))
-    return;
-  face2(inCanvas2, result.face, localOptions);
-  body2(inCanvas2, result.body, localOptions);
-  hand2(inCanvas2, result.hand, localOptions);
-  object(inCanvas2, result.object, localOptions);
-  gesture(inCanvas2, result.gesture, localOptions);
+    return null;
+  const promise = Promise.all([
+    face2(inCanvas2, result.face, localOptions),
+    body2(inCanvas2, result.body, localOptions),
+    hand2(inCanvas2, result.hand, localOptions),
+    object(inCanvas2, result.object, localOptions),
+    gesture(inCanvas2, result.gesture, localOptions)
+  ]);
   result.performance.draw = Math.trunc(now() - timestamp);
+  return promise;
 }
 
 // src/persons.ts

dist/human.node.js

@@ -10466,15 +10466,18 @@ async function all(inCanvas2, result, drawOptions) {
   const timestamp = now();
   const localOptions = mergeDeep(options, drawOptions);
   if (!result || !inCanvas2)
-    return;
+    return null;
   if (!(inCanvas2 instanceof HTMLCanvasElement))
-    return;
-  face2(inCanvas2, result.face, localOptions);
-  body2(inCanvas2, result.body, localOptions);
-  hand2(inCanvas2, result.hand, localOptions);
-  object(inCanvas2, result.object, localOptions);
-  gesture(inCanvas2, result.gesture, localOptions);
+    return null;
+  const promise = Promise.all([
+    face2(inCanvas2, result.face, localOptions),
+    body2(inCanvas2, result.body, localOptions),
+    hand2(inCanvas2, result.hand, localOptions),
+    object(inCanvas2, result.object, localOptions),
+    gesture(inCanvas2, result.gesture, localOptions)
+  ]);
   result.performance.draw = Math.trunc(now() - timestamp);
+  return promise;
 }
 
 // src/persons.ts

server/build.log

@@ -1,22 +1,22 @@
-2021-08-14 13:38:09 INFO:  @vladmandic/human version 2.1.3 
-2021-08-14 13:38:09 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0 
-2021-08-14 13:38:09 INFO:  Toolchain:  {"tfjs":"3.8.0","esbuild":"0.12.20","typescript":"4.3.5","typedoc":"0.21.5","eslint":"7.32.0"} 
-2021-08-14 13:38:09 INFO:  Clean: ["dist/*","types/*","typedoc/*"] 
-2021-08-14 13:38:09 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} 
-2021-08-14 13:38:09 STATE:  target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"} 
-2021-08-14 13:38:09 STATE:  target: node type: node: {"imports":42,"importBytes":437182,"outputBytes":378978,"outputFiles":"dist/human.node.js"} 
-2021-08-14 13:38:09 STATE:  target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"} 
-2021-08-14 13:38:09 STATE:  target: nodeGPU type: node: {"imports":42,"importBytes":437190,"outputBytes":378982,"outputFiles":"dist/human.node-gpu.js"} 
-2021-08-14 13:38:09 STATE:  target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"} 
-2021-08-14 13:38:09 STATE:  target: nodeWASM type: node: {"imports":42,"importBytes":437257,"outputBytes":379054,"outputFiles":"dist/human.node-wasm.js"} 
-2021-08-14 13:38:09 STATE:  target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"} 
-2021-08-14 13:38:09 STATE:  target: browserNoBundle type: esm: {"imports":42,"importBytes":437121,"outputBytes":248666,"outputFiles":"dist/human.esm-nobundle.js"} 
-2021-08-14 13:38:10 STATE:  target: browserBundle type: tfjs: {"modules":1170,"moduleBytes":4145868,"imports":7,"importBytes":2168,"outputBytes":2334701,"outputFiles":"dist/tfjs.esm.js"} 
-2021-08-14 13:38:10 STATE:  target: browserBundle type: iife: {"imports":42,"importBytes":2770580,"outputBytes":1379030,"outputFiles":"dist/human.js"} 
-2021-08-14 13:38:11 STATE:  target: browserBundle type: esm: {"imports":42,"importBytes":2770580,"outputBytes":1379022,"outputFiles":"dist/human.esm.js"} 
-2021-08-14 13:38:11 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"] 
-2021-08-14 13:38:33 INFO:  Linter complete: files: 76 errors: 0 warnings: 0 
-2021-08-14 13:38:33 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"] 
-2021-08-14 13:38:33 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"] 
-2021-08-14 13:38:47 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"] 
-2021-08-14 13:39:01 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1 
+2021-08-14 17:59:16 INFO:  @vladmandic/human version 2.1.3 
+2021-08-14 17:59:16 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0 
+2021-08-14 17:59:16 INFO:  Toolchain:  {"tfjs":"3.8.0","esbuild":"0.12.20","typescript":"4.3.5","typedoc":"0.21.5","eslint":"7.32.0"} 
+2021-08-14 17:59:16 INFO:  Clean: ["dist/*","types/*","typedoc/*"] 
+2021-08-14 17:59:16 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} 
+2021-08-14 17:59:16 STATE:  target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"} 
+2021-08-14 17:59:16 STATE:  target: node type: node: {"imports":42,"importBytes":437259,"outputBytes":379053,"outputFiles":"dist/human.node.js"} 
+2021-08-14 17:59:16 STATE:  target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"} 
+2021-08-14 17:59:16 STATE:  target: nodeGPU type: node: {"imports":42,"importBytes":437267,"outputBytes":379057,"outputFiles":"dist/human.node-gpu.js"} 
+2021-08-14 17:59:16 STATE:  target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"} 
+2021-08-14 17:59:16 STATE:  target: nodeWASM type: node: {"imports":42,"importBytes":437334,"outputBytes":379129,"outputFiles":"dist/human.node-wasm.js"} 
+2021-08-14 17:59:16 STATE:  target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"} 
+2021-08-14 17:59:16 STATE:  target: browserNoBundle type: esm: {"imports":42,"importBytes":437198,"outputBytes":248711,"outputFiles":"dist/human.esm-nobundle.js"} 
+2021-08-14 17:59:17 STATE:  target: browserBundle type: tfjs: {"modules":1170,"moduleBytes":4145868,"imports":7,"importBytes":2168,"outputBytes":2334701,"outputFiles":"dist/tfjs.esm.js"} 
+2021-08-14 17:59:17 STATE:  target: browserBundle type: iife: {"imports":42,"importBytes":2770657,"outputBytes":1379075,"outputFiles":"dist/human.js"} 
+2021-08-14 17:59:18 STATE:  target: browserBundle type: esm: {"imports":42,"importBytes":2770657,"outputBytes":1379067,"outputFiles":"dist/human.esm.js"} 
+2021-08-14 17:59:18 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"] 
+2021-08-14 17:59:40 INFO:  Linter complete: files: 76 errors: 0 warnings: 0 
+2021-08-14 17:59:40 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"] 
+2021-08-14 17:59:40 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"] 
+2021-08-14 17:59:54 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"] 
+2021-08-14 18:00:09 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1 

src/draw/draw.ts

@@ -511,16 +511,17 @@ export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasE
 export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptions?: DrawOptions) {
   const timestamp = now();
   const localOptions = mergeDeep(options, drawOptions);
-  if (!result || !inCanvas) return;
-  if (!(inCanvas instanceof HTMLCanvasElement)) return;
-
-  face(inCanvas, result.face, localOptions);
-  body(inCanvas, result.body, localOptions);
-  hand(inCanvas, result.hand, localOptions);
-  object(inCanvas, result.object, localOptions);
-  // person(inCanvas, result.persons, localOptions);
-  gesture(inCanvas, result.gesture, localOptions); // gestures do not have buffering
+  if (!result || !inCanvas) return null;
+  if (!(inCanvas instanceof HTMLCanvasElement)) return null;
 
+  const promise = Promise.all([
+    face(inCanvas, result.face, localOptions),
+    body(inCanvas, result.body, localOptions),
+    hand(inCanvas, result.hand, localOptions),
+    object(inCanvas, result.object, localOptions),
+    // person(inCanvas, result.persons, localOptions);
+    gesture(inCanvas, result.gesture, localOptions), // gestures do not have buffering
+  ]);
   /*
   if (!bufferedResult) bufferedResult = result; // first pass
   else if (localOptions.bufferedOutput) calcBuffered(result); // do results interpolation
@@ -535,4 +536,5 @@ export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptio
   // await Promise.all(promises);
   */
   result.performance.draw = Math.trunc(now() - timestamp);
+  return promise;
 }

types/src/draw/draw.d.ts

@@ -49,4 +49,4 @@ export declare function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, d
 export declare function object(inCanvas: HTMLCanvasElement, result: Array<Item>, drawOptions?: DrawOptions): Promise<void>;
 export declare function person(inCanvas: HTMLCanvasElement, result: Array<Person>, drawOptions?: DrawOptions): Promise<void>;
 export declare function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasElement): Promise<void>;
-export declare function all(inCanvas: HTMLCanvasElement, result: Result, drawOptions?: DrawOptions): Promise<void>;
+export declare function all(inCanvas: HTMLCanvasElement, result: Result, drawOptions?: DrawOptions): Promise<[void, void, void, void, void] | null>;