ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add human.video method

pull/356/head
Vladimir Mandic 2022-09-17 17:19:51 -04:00
parent 1eb5f9b6f4
commit 29736d8b1b
9 changed files with 1164 additions and 1046 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -9,7 +9,7 @@
## Changelog
### **HEAD -> main** 2022/09/13 mandic00@live.com
### **HEAD -> main** 2022/09/14 mandic00@live.com
### **2.10.2** 2022/09/11 mandic00@live.com

20
README.md
View File

@ -64,6 +64,7 @@ JavaScript module using TensorFlow/JS Machine Learning library
- **Full** [[*Live*]](https://vladmandic.github.io/human/demo/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo): Main browser demo app that showcases all Human capabilities
- **Simple** [[*Live*]](https://vladmandic.github.io/human/demo/typescript/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/typescript): Simple demo in WebCam processing demo in TypeScript
- **Embedded** [[*Live*]](https://vladmandic.github.io/human/demo/video/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/video/index.html): Even simpler demo with tiny code embedded in HTML file
- **Face Match** [[*Live*]](https://vladmandic.github.io/human/demo/facematch/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/facematch): Extract faces from images, calculates face descriptors and simmilarities and matches them to known database
- **Face ID** [[*Live*]](https://vladmandic.github.io/human/demo/faceid/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/faceid): Runs multiple checks to validate webcam input before performing face match to faces in IndexDB
- **Multi-thread** [[*Live*]](https://vladmandic.github.io/human/demo/multithread/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/multithread): Runs each Human module in a separate web worker for highest possible performance
@ -321,7 +322,7 @@ async function detectVideo() {
async function drawVideo() {
if (result) { // check if result is available
const interpolated = human.next(result); // calculate next interpolated frame
const interpolated = human.next(result); // get smoothened result using last-known results
human.draw.all(outputCanvas, interpolated); // draw the frame
}
requestAnimationFrame(drawVideo); // run draw loop
@ -331,6 +332,23 @@ detectVideo(); // start detection loop
drawVideo(); // start draw loop
```
or same, but using built-in full video processing instead of running manual frame-by-frame loop:
```js
const human = new Human(); // create instance of Human
const inputVideo = document.getElementById('video-id');
const outputCanvas = document.getElementById('canvas-id');
async function drawResults() {
const interpolated = human.next(); // get smoothened result using last-known results
human.draw.all(outputCanvas, interpolated); // draw the frame
requestAnimationFrame(drawVideo); // run draw loop
}
human.video(inputVideo); // start detection loop which continously updates results
drawResults(); // start draw loop
```
And for even better results, you can run detection in a separate web worker thread
<br><hr><br>

7
TODO.md
View File

@ -43,5 +43,8 @@ Enable via `about:config` -> `gfx.offscreencanvas.enabled`
## Pending Release Changes
- Enable model cache when using web workers
- Improve NodeJS resolver when using ESM
- New API [`human.video()`](https://vladmandic.github.io/human/typedoc/classes/Human.html#video)
Runs continous detection of an input video instead of processing each frame manually
- New simple demo [*Live*](https://vladmandic.github.io/human/demo/video/index.html)
- Enable model cache when using web workers
- Improve NodeJS resolver when using ESM

68
demo/video/index.html Normal file
View File

@ -0,0 +1,68 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Human</title>
<meta name="viewport" content="width=device-width" id="viewport">
<meta name="keywords" content="Human">
<meta name="description" content="Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition; Author: Vladimir Mandic <https://github.com/vladmandic>">
<link rel="manifest" href="../manifest.webmanifest">
<link rel="shortcut icon" href="../../favicon.ico" type="image/x-icon">
<style>
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../../assets/lato-light.woff2') }
body { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; margin: 0; background: black; color: white; overflow: hidden; width: 100vw; height: 100vh; }
</style>
</head>
<body>
<canvas id="canvas" style="margin: 0 auto; width: 100%"></canvas>
<video id="video" playsinline style="display: none"></video>
<pre id="log" style="padding: 8px"></pre>
<script type="module">
import * as H from '../../dist/human.esm.js'; // equivalent of import @vladmandic/Human
const humanConfig = { // user configuration for human, used to fine-tune behavior
modelBasePath: '../../models',
filter: { enabled: true, equalization: false, flip: false },
face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } },
body: { enabled: true },
hand: { enabled: true },
object: { enabled: false },
gesture: { enabled: true },
};
const human = new H.Human(humanConfig); // create instance of human with overrides from user configuration
const video = document.getElementById('video');
const canvas = document.getElementById('canvas');
async function webCam() { // initialize webcam
const options = { audio: false, video: { facingMode: 'user', resizeMode: 'none', width: { ideal: document.body.clientWidth }, height: { ideal: document.body.clientHeight } } }; // set ideal webcam video properties
const stream = await navigator.mediaDevices.getUserMedia(options); // find webcam stream that best matches what we want
const videoReady = new Promise((resolve) => { video.onloadeddata = () => resolve(true); });
video.srcObject = stream; // assign webcam stream to a video element
video.play(); // start webcam
await videoReady; // wait for video ready
canvas.width = video.videoWidth; // set canvas resolution to input webcam native resolution
canvas.height = video.videoHeight;
canvas.onclick = () => { // pause when clicked on screen and resume on next click
if (video.paused) void video.play();
else video.pause();
};
}
async function drawLoop() { // main screen refresh loop
human.draw.canvas(video, canvas); // draw original video to screen canvas // better than using procesed image as this loop happens faster than processing loop
const interpolated = human.next(); // get smoothened result using last-known results
await human.draw.all(canvas, interpolated); // draw labels, boxes, lines, etc.
setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 30 fps
}
async function main() { // main entry point
document.getElementById('log').innerHTML = `human version: ${human.version} | tfjs version: ${human.tf.version['tfjs-core']}<br>platform: ${human.env.platform} | agent ${human.env.agent}<br>`;
await webCam(); // start webcam
human.video(video); // instruct human to detect video frames
await drawLoop(); // start draw loop
}
window.onload = main;
</script>
</body>
</html>

6
package.json
View File

@ -66,9 +66,9 @@
"tensorflow"
],
"devDependencies": {
"@html-eslint/eslint-plugin": "^0.13.2",
"@html-eslint/parser": "^0.13.2",
"@microsoft/api-extractor": "^7.31.0",
"@html-eslint/eslint-plugin": "^0.14.1",
"@html-eslint/parser": "^0.14.0",
"@microsoft/api-extractor": "^7.31.1",
"@tensorflow/tfjs": "^3.20.0",
"@tensorflow/tfjs-backend-cpu": "^3.20.0",
"@tensorflow/tfjs-backend-wasm": "^3.20.0",

29
src/human.ts
View File

@ -551,6 +551,35 @@ export class Human {
resolve(this.result);
});
}
/** Helper function
* @param ms - sleep time in miliseconds
*/
async sleep(ms: number): Promise<void> { // eslint-disable-line class-methods-use-this
return new Promise((resolve) => { setTimeout(resolve, ms); });
}
/** internal structure that keeps track of processed videos @hidden */
#loops: Record<string, boolean> = {};
/** Continously detect video frames
* @param element - HTMLVideoElement input
* @param run - boolean run continously or stop if already running, default true
* @param delay - number delay detection between frames for number of miliseconds, default 0
*/
async video(element: HTMLVideoElement, run: boolean = true, delay: number = 0) {
if (run) {
if (!this.#loops[element.id]) {
if (this.config.debug) log('video start', element.id);
this.#loops[element.id] = true;
}
if (!element.paused && this.#loops[element.id] && (element.readyState >= 2)) await this.detect(element);
if (delay > 0) await this.sleep(delay);
if (this.#loops[element.id]) requestAnimationFrame(() => this.video(element, run, delay));
} else {
if (this.config.debug) log('video stop', element.id);
this.#loops[element.id] = false;
}
}
}
/** Class Human as default export */

78
test/build.log
View File

@ -1,39 +1,39 @@
2022-09-14 11:37:14 DATA:  Build {"name":"@vladmandic/human","version":"2.10.2"}
2022-09-14 11:37:14 INFO:  Application: {"name":"@vladmandic/human","version":"2.10.2"}
2022-09-14 11:37:14 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-09-14 11:37:14 INFO:  Toolchain: {"build":"0.7.12","esbuild":"0.15.7","typescript":"4.8.3","typedoc":"0.23.14","eslint":"8.23.1"}
2022-09-14 11:37:14 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-09-14 11:37:14 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
2022-09-14 11:37:14 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":159,"outputBytes":608}
2022-09-14 11:37:14 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":75,"inputBytes":657017,"outputBytes":308794}
2022-09-14 11:37:14 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":167,"outputBytes":612}
2022-09-14 11:37:14 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":75,"inputBytes":657021,"outputBytes":308798}
2022-09-14 11:37:14 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":206,"outputBytes":664}
2022-09-14 11:37:14 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":75,"inputBytes":657073,"outputBytes":308848}
2022-09-14 11:37:14 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1125,"outputBytes":358}
2022-09-14 11:37:14 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1088,"outputBytes":583}
2022-09-14 11:37:14 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":75,"inputBytes":656992,"outputBytes":307668}
2022-09-14 11:37:14 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1344,"outputBytes":2821914}
2022-09-14 11:37:14 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":75,"inputBytes":3478323,"outputBytes":1687855}
2022-09-14 11:37:15 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":75,"inputBytes":3478323,"outputBytes":3109019}
2022-09-14 11:37:19 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15}
2022-09-14 11:37:21 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":77,"generated":true}
2022-09-14 11:37:21 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6714,"outputBytes":3134}
2022-09-14 11:37:21 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15488,"outputBytes":7788}
2022-09-14 11:37:32 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":110,"errors":0,"warnings":0}
2022-09-14 11:37:32 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-09-14 11:37:32 STATE: Copy: {"input":"tfjs/tfjs.esm.d.ts"}
2022-09-14 11:37:32 INFO:  Done...
2022-09-14 11:37:33 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":193}
2022-09-14 11:37:33 STATE: Copy: {"input":"types/human.d.ts"}
2022-09-14 11:37:33 INFO:  Analyze models: {"folders":8,"result":"models/models.json"}
2022-09-14 11:37:33 STATE: Models {"folder":"./models","models":13}
2022-09-14 11:37:33 STATE: Models {"folder":"../human-models/models","models":42}
2022-09-14 11:37:33 STATE: Models {"folder":"../blazepose/model/","models":4}
2022-09-14 11:37:33 STATE: Models {"folder":"../anti-spoofing/model","models":1}
2022-09-14 11:37:33 STATE: Models {"folder":"../efficientpose/models","models":3}
2022-09-14 11:37:33 STATE: Models {"folder":"../insightface/models","models":5}
2022-09-14 11:37:33 STATE: Models {"folder":"../movenet/models","models":3}
2022-09-14 11:37:33 STATE: Models {"folder":"../nanodet/models","models":4}
2022-09-14 11:37:34 STATE: Models: {"count":57,"totalSize":383017442}
2022-09-14 11:37:34 INFO:  Human Build complete... {"logFile":"test/build.log"}
2022-09-17 17:17:01 DATA:  Build {"name":"@vladmandic/human","version":"2.10.2"}
2022-09-17 17:17:01 INFO:  Application: {"name":"@vladmandic/human","version":"2.10.2"}
2022-09-17 17:17:01 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-09-17 17:17:01 INFO:  Toolchain: {"build":"0.7.12","esbuild":"0.15.7","typescript":"4.8.3","typedoc":"0.23.14","eslint":"8.23.1"}
2022-09-17 17:17:01 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-09-17 17:17:01 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
2022-09-17 17:17:01 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":159,"outputBytes":608}
2022-09-17 17:17:01 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":75,"inputBytes":658253,"outputBytes":309214}
2022-09-17 17:17:01 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":167,"outputBytes":612}
2022-09-17 17:17:01 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":75,"inputBytes":658257,"outputBytes":309218}
2022-09-17 17:17:01 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":206,"outputBytes":664}
2022-09-17 17:17:01 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":75,"inputBytes":658309,"outputBytes":309268}
2022-09-17 17:17:01 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1125,"outputBytes":358}
2022-09-17 17:17:01 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1088,"outputBytes":583}
2022-09-17 17:17:01 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":75,"inputBytes":658228,"outputBytes":308088}
2022-09-17 17:17:02 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1344,"outputBytes":2821914}
2022-09-17 17:17:02 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":75,"inputBytes":3479559,"outputBytes":1688279}
2022-09-17 17:17:02 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":75,"inputBytes":3479559,"outputBytes":3109891}
2022-09-17 17:17:06 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15}
2022-09-17 17:17:09 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":77,"generated":true}
2022-09-17 17:17:09 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6714,"outputBytes":3134}
2022-09-17 17:17:09 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15488,"outputBytes":7788}
2022-09-17 17:17:20 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":110,"errors":0,"warnings":0}
2022-09-17 17:17:20 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-09-17 17:17:20 STATE: Copy: {"input":"tfjs/tfjs.esm.d.ts"}
2022-09-17 17:17:20 INFO:  Done...
2022-09-17 17:17:21 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":193}
2022-09-17 17:17:21 STATE: Copy: {"input":"types/human.d.ts"}
2022-09-17 17:17:21 INFO:  Analyze models: {"folders":8,"result":"models/models.json"}
2022-09-17 17:17:21 STATE: Models {"folder":"./models","models":13}
2022-09-17 17:17:21 STATE: Models {"folder":"../human-models/models","models":42}
2022-09-17 17:17:21 STATE: Models {"folder":"../blazepose/model/","models":4}
2022-09-17 17:17:21 STATE: Models {"folder":"../anti-spoofing/model","models":1}
2022-09-17 17:17:21 STATE: Models {"folder":"../efficientpose/models","models":3}
2022-09-17 17:17:21 STATE: Models {"folder":"../insightface/models","models":5}
2022-09-17 17:17:21 STATE: Models {"folder":"../movenet/models","models":3}
2022-09-17 17:17:21 STATE: Models {"folder":"../nanodet/models","models":4}
2022-09-17 17:17:22 STATE: Models: {"count":57,"totalSize":383017442}
2022-09-17 17:17:22 INFO:  Human Build complete... {"logFile":"test/build.log"}

1998
test/test.log
View File

File diff suppressed because it is too large Load Diff

2
wiki

@ -1 +1 @@
Subproject commit 9785618d94c8eb325fefbf8ca1212359b5c913d6
Subproject commit 7a9a68dc4dbbc06c82324266c9a31d9fe037cf3b