From f29d85dacdd811ec6ee07635c3d1e68415ca730d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 14 Aug 2021 18:00:26 -0400 Subject: [PATCH] experimental webgpu support --- .gitignore | 1 + CHANGELOG.md | 1 + demo/helpers/menu.js | 6 +++--- demo/index.js | 42 +++++++++++++++++++++++------------------- src/draw/draw.ts | 20 +++++++++++--------- 5 files changed, 39 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index ee89780a..25d22f5c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules pnpm-lock.yaml +assets/tf* diff --git a/CHANGELOG.md b/CHANGELOG.md index 75779336..33043700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Repository: **** ### **HEAD -> main** 2021/08/14 mandic00@live.com +- add backend initialization checks - complete async work - list detect cameras - switch to async data reads diff --git a/demo/helpers/menu.js b/demo/helpers/menu.js index b1427155..cd891cdc 100644 --- a/demo/helpers/menu.js +++ b/demo/helpers/menu.js @@ -21,7 +21,7 @@ function createCSS() { if (CSScreated) return; const css = ` :root { --rounded: 0.1rem; } - .menu { position: absolute; top: 0rem; right: 0; min-width: 180px; width: max-content; padding: 0.2rem 0.2rem 0 0.2rem; line-height: 1.8rem; z-index: 10; background: ${theme.background}; border: none } + .menu { position: absolute; top: 0rem; right: 0; min-width: 180px; width: max-content; padding: 0.2rem 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; background: ${theme.background}; border: none } .button { text-shadow: none; } .menu-container { display: block; max-height: 100vh; } @@ -46,7 +46,7 @@ function createCSS() { .menu-button:hover { background: ${theme.buttonHover}; box-shadow: 4px 4px 4px 0 black; } .menu-button:focus { outline: none; } - .menu-checkbox { width: 2.6rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 0.5rem 0 0; position: relative; border-radius: var(--rounded); } + .menu-checkbox { width: 2.6rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 1.0rem 0 0; position: relative; border-radius: var(--rounded); } .menu-checkbox:after { content: 'OFF'; color: ${theme.checkboxOff}; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; } .menu-checkbox:before { content: 'ON'; color: ${theme.checkboxOn}; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; } .menu-checkbox-label { width: 1.3rem; height: 1rem; cursor: pointer; position: absolute; top: 0; left: 0rem; z-index: 1; background: ${theme.checkboxOff}; @@ -55,7 +55,7 @@ function createCSS() { input[type=checkbox] { visibility: hidden; } input[type=checkbox]:checked + label { left: 1.4rem; background: ${theme.checkboxOn}; } - .menu-range { margin: 0.2rem 0.5rem 0 0; width: 3.5rem; background: transparent; color: ${theme.rangeBackground}; } + .menu-range { margin: 0.2rem 1.0rem 0 0; width: 5rem; background: transparent; color: ${theme.rangeBackground}; } .menu-range:before { color: ${theme.rangeLabel}; margin: 0 0.4rem 0 0; font-weight: 800; font-size: 0.6rem; position: relative; top: 0.3rem; content: attr(value); } input[type=range] { -webkit-appearance: none; } diff --git a/demo/index.js b/demo/index.js index 8ffa3d40..9aec5489 100644 --- a/demo/index.js +++ b/demo/index.js @@ -199,6 +199,13 @@ async function calcSimmilarity(result) { document.getElementById('similarity').innerText = `similarity: ${Math.trunc(1000 * similarity) / 10}%`; } +const isLive = (input) => { + const videoLive = input.readyState > 2; + const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live'; + const live = (videoLive || cameraLive) && (!input.paused); + return live; +}; + // draws processed results and starts processing of a next frame let lastDraw = performance.now(); async function drawResults(input) { @@ -271,11 +278,17 @@ async function drawResults(input) { ui.lastFrame = performance.now(); // if buffered, immediate loop but limit frame rate although it's going to run slower as JS is singlethreaded if (ui.buffered) { - ui.drawThread = requestAnimationFrame(() => drawResults(input)); + if (isLive(input)) { + ui.drawThread = requestAnimationFrame(() => drawResults(input)); + } else { + cancelAnimationFrame(ui.drawThread); + ui.drawThread = null; + } } else { if (ui.drawThread) { log('stopping buffered refresh'); cancelAnimationFrame(ui.drawThread); + ui.drawThread = null; } } } @@ -326,7 +339,7 @@ async function setupCamera() { }; // enumerate devices for diag purposes if (initialCameraAccess) { - navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated devices:', devices)); + navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated input devices:', devices)); log('camera constraints', constraints); } try { @@ -365,7 +378,6 @@ async function setupCamera() { // eslint-disable-next-line no-use-before-define if (live && !ui.detectThread) runHumanDetect(video, canvas); ui.busy = false; - status(); resolve(); }; }); @@ -402,6 +414,7 @@ function webWorker(input, image, canvas, timestamp) { worker = new Worker(ui.worker); // after receiving message from webworker, parse&draw results and send new frame for processing worker.addEventListener('message', (msg) => { + status(); if (msg.data.result.performance && msg.data.result.performance.total) ui.detectFPS.push(1000 / msg.data.result.performance.total); if (ui.detectFPS.length > ui.maxFPSframes) ui.detectFPS.shift(); if (ui.bench) { @@ -421,14 +434,8 @@ function webWorker(input, image, canvas, timestamp) { } ui.framesDetect++; - if (!ui.drawThread) { - status(); - drawResults(input); - } - const videoLive = (input.readyState > 2) && (!input.paused); - const cameraLive = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live') && !input.paused; - const live = videoLive || cameraLive; - if (live) { + if (!ui.drawThread) drawResults(input); + if (isLive(input)) { // eslint-disable-next-line no-use-before-define ui.detectThread = requestAnimationFrame((now) => runHumanDetect(input, canvas, now)); } @@ -441,16 +448,13 @@ function webWorker(input, image, canvas, timestamp) { // main processing function when input is webcam, can use direct invocation or web worker function runHumanDetect(input, canvas, timestamp) { // if live video - const videoLive = input.readyState > 2; - const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live'; - const live = (videoLive || cameraLive) && (!input.paused); - if (!live) { + if (!isLive(input)) { // stop ui refresh // if (ui.drawThread) cancelAnimationFrame(ui.drawThread); if (ui.detectThread) cancelAnimationFrame(ui.detectThread); - // if we want to continue and camera not ready, retry in 0.5sec, else just give up if (input.paused) log('video paused'); - else if (cameraLive && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500); + // if we want to continue and camera not ready, retry in 0.5sec, else just give up + // else if (cameraLive && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500); else log(`video not ready: track state: ${input.srcObject ? input.srcObject.getVideoTracks()[0].readyState : 'unknown'} stream state: ${input.readyState}`); log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw); log('memory', human.tf.engine().memory()); @@ -600,7 +604,6 @@ async function detectVideo() { document.getElementById('btnStartText').innerHTML = 'pause video'; await video.play(); runHumanDetect(video, canvas); - status(); } else { status(cameraError); } @@ -721,6 +724,8 @@ function setupMenu() { compare.original = null; }); + for (const m of Object.values(menu)) m.hide(); + document.getElementById('btnDisplay').addEventListener('click', (evt) => menu.display.toggle(evt)); document.getElementById('btnImage').addEventListener('click', (evt) => menu.image.toggle(evt)); document.getElementById('btnProcess').addEventListener('click', (evt) => menu.process.toggle(evt)); @@ -974,7 +979,6 @@ async function main() { status('human: ready'); document.getElementById('loader').style.display = 'none'; document.getElementById('play').style.display = 'block'; - for (const m of Object.values(menu)) m.hide(); // init drag & drop await dragAndDrop(); diff --git a/src/draw/draw.ts b/src/draw/draw.ts index beeb6fde..78406396 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -511,16 +511,17 @@ export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasE export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptions?: DrawOptions) { const timestamp = now(); const localOptions = mergeDeep(options, drawOptions); - if (!result || !inCanvas) return; - if (!(inCanvas instanceof HTMLCanvasElement)) return; - - face(inCanvas, result.face, localOptions); - body(inCanvas, result.body, localOptions); - hand(inCanvas, result.hand, localOptions); - object(inCanvas, result.object, localOptions); - // person(inCanvas, result.persons, localOptions); - gesture(inCanvas, result.gesture, localOptions); // gestures do not have buffering + if (!result || !inCanvas) return null; + if (!(inCanvas instanceof HTMLCanvasElement)) return null; + const promise = Promise.all([ + face(inCanvas, result.face, localOptions), + body(inCanvas, result.body, localOptions), + hand(inCanvas, result.hand, localOptions), + object(inCanvas, result.object, localOptions), + // person(inCanvas, result.persons, localOptions); + gesture(inCanvas, result.gesture, localOptions), // gestures do not have buffering + ]); /* if (!bufferedResult) bufferedResult = result; // first pass else if (localOptions.bufferedOutput) calcBuffered(result); // do results interpolation @@ -535,4 +536,5 @@ export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptio // await Promise.all(promises); */ result.performance.draw = Math.trunc(now() - timestamp); + return promise; }