ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

experimental webgpu support

pull/356/head
Vladimir Mandic 2021-08-14 18:00:26 -04:00
parent f867d46b85
commit f29d85dacd
5 changed files with 39 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
node_modules
pnpm-lock.yaml
assets/tf*

1
CHANGELOG.md
View File

@ -11,6 +11,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/08/14 mandic00@live.com
- add backend initialization checks
- complete async work
- list detect cameras
- switch to async data reads

6
demo/helpers/menu.js
View File

@ -21,7 +21,7 @@ function createCSS() {
if (CSScreated) return;
const css = `
:root { --rounded: 0.1rem; }
.menu { position: absolute; top: 0rem; right: 0; min-width: 180px; width: max-content; padding: 0.2rem 0.2rem 0 0.2rem; line-height: 1.8rem; z-index: 10; background: ${theme.background}; border: none }
.menu { position: absolute; top: 0rem; right: 0; min-width: 180px; width: max-content; padding: 0.2rem 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; background: ${theme.background}; border: none }
.button { text-shadow: none; }
.menu-container { display: block; max-height: 100vh; }
@ -46,7 +46,7 @@ function createCSS() {
.menu-button:hover { background: ${theme.buttonHover}; box-shadow: 4px 4px 4px 0 black; }
.menu-button:focus { outline: none; }
.menu-checkbox { width: 2.6rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 0.5rem 0 0; position: relative; border-radius: var(--rounded); }
.menu-checkbox { width: 2.6rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 1.0rem 0 0; position: relative; border-radius: var(--rounded); }
.menu-checkbox:after { content: 'OFF'; color: ${theme.checkboxOff}; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
.menu-checkbox:before { content: 'ON'; color: ${theme.checkboxOn}; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
.menu-checkbox-label { width: 1.3rem; height: 1rem; cursor: pointer; position: absolute; top: 0; left: 0rem; z-index: 1; background: ${theme.checkboxOff};
@ -55,7 +55,7 @@ function createCSS() {
input[type=checkbox] { visibility: hidden; }
input[type=checkbox]:checked + label { left: 1.4rem; background: ${theme.checkboxOn}; }
.menu-range { margin: 0.2rem 0.5rem 0 0; width: 3.5rem; background: transparent; color: ${theme.rangeBackground}; }
.menu-range { margin: 0.2rem 1.0rem 0 0; width: 5rem; background: transparent; color: ${theme.rangeBackground}; }
.menu-range:before { color: ${theme.rangeLabel}; margin: 0 0.4rem 0 0; font-weight: 800; font-size: 0.6rem; position: relative; top: 0.3rem; content: attr(value); }
input[type=range] { -webkit-appearance: none; }

42
demo/index.js
View File

@ -199,6 +199,13 @@ async function calcSimmilarity(result) {
document.getElementById('similarity').innerText = `similarity: ${Math.trunc(1000 * similarity) / 10}%`;
}
const isLive = (input) => {
const videoLive = input.readyState > 2;
const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live';
const live = (videoLive || cameraLive) && (!input.paused);
return live;
};
// draws processed results and starts processing of a next frame
let lastDraw = performance.now();
async function drawResults(input) {
@ -271,11 +278,17 @@ async function drawResults(input) {
ui.lastFrame = performance.now();
// if buffered, immediate loop but limit frame rate although it's going to run slower as JS is singlethreaded
if (ui.buffered) {
ui.drawThread = requestAnimationFrame(() => drawResults(input));
if (isLive(input)) {
ui.drawThread = requestAnimationFrame(() => drawResults(input));
} else {
cancelAnimationFrame(ui.drawThread);
ui.drawThread = null;
}
} else {
if (ui.drawThread) {
log('stopping buffered refresh');
cancelAnimationFrame(ui.drawThread);
ui.drawThread = null;
}
}
}
@ -326,7 +339,7 @@ async function setupCamera() {
};
// enumerate devices for diag purposes
if (initialCameraAccess) {
navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated devices:', devices));
navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated input devices:', devices));
log('camera constraints', constraints);
}
try {
@ -365,7 +378,6 @@ async function setupCamera() {
// eslint-disable-next-line no-use-before-define
if (live && !ui.detectThread) runHumanDetect(video, canvas);
ui.busy = false;
status();
resolve();
};
});
@ -402,6 +414,7 @@ function webWorker(input, image, canvas, timestamp) {
worker = new Worker(ui.worker);
// after receiving message from webworker, parse&draw results and send new frame for processing
worker.addEventListener('message', (msg) => {
status();
if (msg.data.result.performance && msg.data.result.performance.total) ui.detectFPS.push(1000 / msg.data.result.performance.total);
if (ui.detectFPS.length > ui.maxFPSframes) ui.detectFPS.shift();
if (ui.bench) {
@ -421,14 +434,8 @@ function webWorker(input, image, canvas, timestamp) {
}
ui.framesDetect++;
if (!ui.drawThread) {
status();
drawResults(input);
}
const videoLive = (input.readyState > 2) && (!input.paused);
const cameraLive = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live') && !input.paused;
const live = videoLive || cameraLive;
if (live) {
if (!ui.drawThread) drawResults(input);
if (isLive(input)) {
// eslint-disable-next-line no-use-before-define
ui.detectThread = requestAnimationFrame((now) => runHumanDetect(input, canvas, now));
}
@ -441,16 +448,13 @@ function webWorker(input, image, canvas, timestamp) {
// main processing function when input is webcam, can use direct invocation or web worker
function runHumanDetect(input, canvas, timestamp) {
// if live video
const videoLive = input.readyState > 2;
const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live';
const live = (videoLive || cameraLive) && (!input.paused);
if (!live) {
if (!isLive(input)) {
// stop ui refresh
// if (ui.drawThread) cancelAnimationFrame(ui.drawThread);
if (ui.detectThread) cancelAnimationFrame(ui.detectThread);
// if we want to continue and camera not ready, retry in 0.5sec, else just give up
if (input.paused) log('video paused');
else if (cameraLive && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500);
// if we want to continue and camera not ready, retry in 0.5sec, else just give up
// else if (cameraLive && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500);
else log(`video not ready: track state: ${input.srcObject ? input.srcObject.getVideoTracks()[0].readyState : 'unknown'} stream state: ${input.readyState}`);
log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw);
log('memory', human.tf.engine().memory());
@ -600,7 +604,6 @@ async function detectVideo() {
document.getElementById('btnStartText').innerHTML = 'pause video';
await video.play();
runHumanDetect(video, canvas);
status();
} else {
status(cameraError);
}
@ -721,6 +724,8 @@ function setupMenu() {
compare.original = null;
});
for (const m of Object.values(menu)) m.hide();
document.getElementById('btnDisplay').addEventListener('click', (evt) => menu.display.toggle(evt));
document.getElementById('btnImage').addEventListener('click', (evt) => menu.image.toggle(evt));
document.getElementById('btnProcess').addEventListener('click', (evt) => menu.process.toggle(evt));
@ -974,7 +979,6 @@ async function main() {
status('human: ready');
document.getElementById('loader').style.display = 'none';
document.getElementById('play').style.display = 'block';
for (const m of Object.values(menu)) m.hide();
// init drag & drop
await dragAndDrop();

20
src/draw/draw.ts
View File

@ -511,16 +511,17 @@ export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasE
export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptions?: DrawOptions) {
const timestamp = now();
const localOptions = mergeDeep(options, drawOptions);
if (!result || !inCanvas) return;
if (!(inCanvas instanceof HTMLCanvasElement)) return;
face(inCanvas, result.face, localOptions);
body(inCanvas, result.body, localOptions);
hand(inCanvas, result.hand, localOptions);
object(inCanvas, result.object, localOptions);
// person(inCanvas, result.persons, localOptions);
gesture(inCanvas, result.gesture, localOptions); // gestures do not have buffering
if (!result || !inCanvas) return null;
if (!(inCanvas instanceof HTMLCanvasElement)) return null;
const promise = Promise.all([
face(inCanvas, result.face, localOptions),
body(inCanvas, result.body, localOptions),
hand(inCanvas, result.hand, localOptions),
object(inCanvas, result.object, localOptions),
// person(inCanvas, result.persons, localOptions);
gesture(inCanvas, result.gesture, localOptions), // gestures do not have buffering
]);
/*
if (!bufferedResult) bufferedResult = result; // first pass
else if (localOptions.bufferedOutput) calcBuffered(result); // do results interpolation
@ -535,4 +536,5 @@ export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptio
// await Promise.all(promises);
*/
result.performance.draw = Math.trunc(now() - timestamp);
return promise;
}