ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add experimental webgu demo

pull/193/head
Vladimir Mandic 2021-08-14 13:39:26 -04:00
parent eaba4d1d45
commit 716924c383
13 changed files with 189 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -11,9 +11,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/08/14 mandic00@live.com
### **origin/main** 2021/08/13 mandic00@live.com
- complete async work
- list detect cameras
- switch to async data reads

33
demo/webgpu/index.html Normal file
View File

@ -0,0 +1,33 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Human</title>
<meta name="viewport" content="width=device-width" id="viewport">
<meta name="keywords" content="Human">
<meta name="application-name" content="Human">
<meta name="description" content="Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition; Author: Vladimir Mandic <https://github.com/vladmandic>">
<meta name="msapplication-tooltip" content="Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition; Author: Vladimir Mandic <https://github.com/vladmandic>">
<meta name="theme-color" content="#000000">
<link rel="manifest" href="../manifest.webmanifest">
<link rel="shortcut icon" href="../../favicon.ico" type="image/x-icon">
<link rel="apple-touch-icon" href="../../assets/icon.png">
<script src="./index.js" type="module"></script>
<style>
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../../assets/lato-light.woff2') }
html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
body { margin: 0; background: black; color: white; overflow-x: hidden; width: 100vw; height: 100vh; }
body::-webkit-scrollbar { display: none; }
.status { position: absolute; width: 100vw; bottom: 10%; text-align: center; font-size: 3rem; font-weight: 100; text-shadow: 2px 2px #303030; }
.log { position: absolute; bottom: 0; margin: 0.4rem 0.4rem 0 0.4rem; font-size: 0.9rem; }
.video { display: none; }
.canvas { margin: 0 auto; }
</style>
</head>
<body>
<div id="status" class="status"></div>
<canvas id="canvas" class="canvas"></canvas>
<video id="video" playsinline class="video"></video>
<div id="log" class="log"></div>
</body>
</html>

119
demo/webgpu/index.js Normal file
View File

@ -0,0 +1,119 @@
/**
* Human demo for browsers
*
* @description Experimental Demo app for Human using WebGPU
*
*/
// @ts-nocheck // typescript checks disabled as this is pure javascript
import '../../node_modules/@tensorflow/tfjs-core/dist/tf-core.es2017.js';
import '../../node_modules/@tensorflow/tfjs-backend-webgpu/dist/tf-backend-webgpu.es2017.js';
import Human from '../../dist/human.esm.js';
let human;
let canvas;
let video;
let result;
const myConfig = {
backend: 'webgpu',
async: false,
warmup: 'none',
modelBasePath: '../../models',
cacheSensitivity: 0,
filter: {
enabled: false,
flip: false,
},
face: { enabled: false,
detector: { return: false, rotation: false },
mesh: { enabled: false },
iris: { enabled: false },
description: { enabled: false },
emotion: { enabled: false },
},
object: { enabled: false },
gesture: { enabled: false },
hand: { enabled: false },
body: { enabled: true },
segmentation: { enabled: false },
};
const time = {
detect: 0,
draw: 0,
};
function log(...msg) {
const dt = new Date();
const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`;
// eslint-disable-next-line no-console
console.log(ts, ...msg);
}
async function drawResults() {
const interpolated = human.next(result);
await human.draw.all(canvas, interpolated);
document.getElementById('log').innerText = `Human: version ${human.version} | FPS: ${1000 / time.detect} / ${1000 / time.draw}`;
requestAnimationFrame(drawResults);
}
async function runDetection() {
result = await human.detect(video);
requestAnimationFrame(runDetection);
}
async function setupCamera() {
video = document.getElementById('video');
canvas = document.getElementById('canvas');
const output = document.getElementById('log');
let stream;
const constraints = {
audio: false,
video: {
facingMode: 'user',
resizeMode: 'crop-and-scale',
width: { ideal: document.body.clientWidth },
// height: { ideal: document.body.clientHeight }, // not set as we're using aspectRation to get height instead
aspectRatio: document.body.clientWidth / document.body.clientHeight,
},
};
// enumerate devices for diag purposes
navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated devices:', devices));
log('camera constraints', constraints);
try {
stream = await navigator.mediaDevices.getUserMedia(constraints);
} catch (err) {
output.innerText += `\n${err.name}: ${err.message}`;
status(err.name);
log('camera error:', err);
}
const tracks = stream.getVideoTracks();
log('enumerated viable tracks:', tracks);
const track = stream.getVideoTracks()[0];
const settings = track.getSettings();
log('selected video source:', track, settings);
const promise = !stream || new Promise((resolve) => {
video.onloadeddata = () => {
if (settings.width > settings.height) canvas.style.width = '100vw';
else canvas.style.height = '100vh';
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
video.play();
resolve();
};
});
// attach input to video element
if (stream) video.srcObject = stream;
return promise;
}
async function main() {
human = new Human(myConfig);
document.getElementById('log').innerText = `Human: version ${human.version}`;
await setupCamera();
runDetection();
drawResults();
}
window.onload = main;

2
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/human.node-gpu.js vendored
View File

@ -10539,6 +10539,8 @@ function join2(faces, bodies, hands, gestures, shape) {
var bufferedResult = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
function calc(newResult) {
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u;
if (!newResult)
return { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
const elapsed = Date.now() - newResult.timestamp;
const bufferedFactor = elapsed < 1e3 ? 8 - Math.log(elapsed) : 1;
bufferedResult.canvas = newResult.canvas;

2
dist/human.node-wasm.js vendored
View File

@ -10540,6 +10540,8 @@ function join2(faces, bodies, hands, gestures, shape) {
var bufferedResult = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
function calc(newResult) {
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u;
if (!newResult)
return { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
const elapsed = Date.now() - newResult.timestamp;
const bufferedFactor = elapsed < 1e3 ? 8 - Math.log(elapsed) : 1;
bufferedResult.canvas = newResult.canvas;

2
dist/human.node.js vendored
View File

@ -10539,6 +10539,8 @@ function join2(faces, bodies, hands, gestures, shape) {
var bufferedResult = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
function calc(newResult) {
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u;
if (!newResult)
return { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
const elapsed = Date.now() - newResult.timestamp;
const bufferedFactor = elapsed < 1e3 ? 8 - Math.log(elapsed) : 1;
bufferedResult.canvas = newResult.canvas;

44
server/build.log
View File

@ -1,22 +1,22 @@
2021-08-14 11:16:37 INFO:  @vladmandic/human version 2.1.3
2021-08-14 11:16:37 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0
2021-08-14 11:16:37 INFO:  Toolchain: {"tfjs":"3.8.0","esbuild":"0.12.20","typescript":"4.3.5","typedoc":"0.21.5","eslint":"7.32.0"}
2021-08-14 11:16:37 INFO:  Clean: ["dist/*","types/*","typedoc/*"]
2021-08-14 11:16:37 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-08-14 11:16:37 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 11:16:37 STATE: target: node type: node: {"imports":42,"importBytes":437054,"outputBytes":378846,"outputFiles":"dist/human.node.js"}
2021-08-14 11:16:37 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 11:16:37 STATE: target: nodeGPU type: node: {"imports":42,"importBytes":437062,"outputBytes":378850,"outputFiles":"dist/human.node-gpu.js"}
2021-08-14 11:16:37 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 11:16:37 STATE: target: nodeWASM type: node: {"imports":42,"importBytes":437129,"outputBytes":378922,"outputFiles":"dist/human.node-wasm.js"}
2021-08-14 11:16:37 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 11:16:37 STATE: target: browserNoBundle type: esm: {"imports":42,"importBytes":436993,"outputBytes":248569,"outputFiles":"dist/human.esm-nobundle.js"}
2021-08-14 11:16:38 STATE: target: browserBundle type: tfjs: {"modules":1170,"moduleBytes":4145868,"imports":7,"importBytes":2168,"outputBytes":2334701,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 11:16:38 STATE: target: browserBundle type: iife: {"imports":42,"importBytes":2770452,"outputBytes":1378933,"outputFiles":"dist/human.js"}
2021-08-14 11:16:39 STATE: target: browserBundle type: esm: {"imports":42,"importBytes":2770452,"outputBytes":1378925,"outputFiles":"dist/human.esm.js"}
2021-08-14 11:16:39 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"]
2021-08-14 11:17:01 INFO:  Linter complete: files: 75 errors: 0 warnings: 0
2021-08-14 11:17:01 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-08-14 11:17:01 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"]
2021-08-14 11:17:15 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
2021-08-14 11:17:28 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1
2021-08-14 13:38:09 INFO:  @vladmandic/human version 2.1.3
2021-08-14 13:38:09 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0
2021-08-14 13:38:09 INFO:  Toolchain: {"tfjs":"3.8.0","esbuild":"0.12.20","typescript":"4.3.5","typedoc":"0.21.5","eslint":"7.32.0"}
2021-08-14 13:38:09 INFO:  Clean: ["dist/*","types/*","typedoc/*"]
2021-08-14 13:38:09 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-08-14 13:38:09 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 13:38:09 STATE: target: node type: node: {"imports":42,"importBytes":437182,"outputBytes":378978,"outputFiles":"dist/human.node.js"}
2021-08-14 13:38:09 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 13:38:09 STATE: target: nodeGPU type: node: {"imports":42,"importBytes":437190,"outputBytes":378982,"outputFiles":"dist/human.node-gpu.js"}
2021-08-14 13:38:09 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 13:38:09 STATE: target: nodeWASM type: node: {"imports":42,"importBytes":437257,"outputBytes":379054,"outputFiles":"dist/human.node-wasm.js"}
2021-08-14 13:38:09 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 13:38:09 STATE: target: browserNoBundle type: esm: {"imports":42,"importBytes":437121,"outputBytes":248666,"outputFiles":"dist/human.esm-nobundle.js"}
2021-08-14 13:38:10 STATE: target: browserBundle type: tfjs: {"modules":1170,"moduleBytes":4145868,"imports":7,"importBytes":2168,"outputBytes":2334701,"outputFiles":"dist/tfjs.esm.js"}
2021-08-14 13:38:10 STATE: target: browserBundle type: iife: {"imports":42,"importBytes":2770580,"outputBytes":1379030,"outputFiles":"dist/human.js"}
2021-08-14 13:38:11 STATE: target: browserBundle type: esm: {"imports":42,"importBytes":2770580,"outputBytes":1379022,"outputFiles":"dist/human.esm.js"}
2021-08-14 13:38:11 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"]
2021-08-14 13:38:33 INFO:  Linter complete: files: 76 errors: 0 warnings: 0
2021-08-14 13:38:33 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-08-14 13:38:33 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"]
2021-08-14 13:38:47 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
2021-08-14 13:39:01 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1

1
src/interpolate.ts
View File

@ -7,6 +7,7 @@ import type { Result, Face, Body, Hand, Item, Gesture, Person } from './result';
const bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
export function calc(newResult: Result): Result {
if (!newResult) return { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
// each record is only updated using deep clone when number of detected record changes, otherwise it will converge by itself
// otherwise bufferedResult is a shallow clone of result plus updated local calculated values
// thus mixing by-reference and by-value assignments to minimize memory operations