ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

major update for 1.8 release candidate

pull/134/head
Vladimir Mandic 2021-04-25 13:16:04 -04:00
parent 05df95fe4f
commit 66054542c2
46 changed files with 836 additions and 1096 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@ -1,6 +1,6 @@
# @vladmandic/human # @vladmandic/human
Version: **1.7.1** Version: **1.8.0**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>** Author: **Vladimir Mandic <mandic00@live.com>**
@ -9,11 +9,12 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog ## Changelog
### **HEAD -> main** 2021/04/25 mandic00@live.com
### **1.7.1** 2021/04/25 mandic00@live.com ### **1.7.1** 2021/04/25 mandic00@live.com
- remove obsolete binary models
### **origin/main** 2021/04/24 mandic00@live.com
- enable cross origin isolation - enable cross origin isolation
- rewrite posenet decoder - rewrite posenet decoder
- remove efficientpose - remove efficientpose

19
TODO.md
View File

@ -17,4 +17,21 @@ N/A
- Blazepose - Blazepose
Needs detector before running pose to center the image Needs detector before running pose to center the image
## Soon to be Removed ## RC: 1.8
### Done
Major configuration simplification:
- Unified minConfidence and scoreThresdold as minConfidence
- Replaced nmsRadius with built-in default
- Replaced maxFaces, maxDetections, maxHands, maxResults with maxDetected
- Remove deallocate, profile, scoped
Stop building sourcemaps for NodeJS deliverables
### TBD
- Remove modelPaths
- Remove blazeface-front, replace blazeface-back with blazeface
- NodeJS Exception handling

2
demo/face3d.js
View File

@ -12,7 +12,7 @@ const userConfig = {
videoOptimized: true, videoOptimized: true,
filter: { enabled: false }, filter: { enabled: false },
face: { enabled: true, face: { enabled: true,
detector: { rotation: false, maxFaces: 1 }, detector: { rotation: false, maxDetected: 1 },
mesh: { enabled: true }, mesh: { enabled: true },
iris: { enabled: true }, iris: { enabled: true },
description: { enabled: false }, description: { enabled: false },

15
demo/index.js
View File

@ -492,15 +492,13 @@ function setupMenu() {
menu.process = new Menu(document.body, '', { top, left: x[2] }); menu.process = new Menu(document.body, '', { top, left: x[2] });
menu.process.addList('backend', ['cpu', 'webgl', 'wasm', 'humangl'], human.config.backend, (val) => human.config.backend = val); menu.process.addList('backend', ['cpu', 'webgl', 'wasm', 'humangl'], human.config.backend, (val) => human.config.backend = val);
menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val); menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val);
// menu.process.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val);
// menu.process.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val);
menu.process.addBool('use web worker', ui, 'useWorker'); menu.process.addBool('use web worker', ui, 'useWorker');
menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">'); menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.process.addLabel('model parameters'); menu.process.addLabel('model parameters');
menu.process.addRange('max objects', human.config.face.detector, 'maxFaces', 1, 50, 1, (val) => { menu.process.addRange('max objects', human.config.face.detector, 'maxDetected', 1, 50, 1, (val) => {
human.config.face.detector.maxFaces = parseInt(val); human.config.face.detector.maxDetected = parseInt(val);
human.config.body.maxDetections = parseInt(val); human.config.body.maxDetected = parseInt(val);
human.config.hand.maxHands = parseInt(val); human.config.hand.maxDetected = parseInt(val);
}); });
menu.process.addRange('skip frames', human.config.face.detector, 'skipFrames', 0, 50, 1, (val) => { menu.process.addRange('skip frames', human.config.face.detector, 'skipFrames', 0, 50, 1, (val) => {
human.config.face.detector.skipFrames = parseInt(val); human.config.face.detector.skipFrames = parseInt(val);
@ -512,11 +510,6 @@ function setupMenu() {
human.config.face.emotion.minConfidence = parseFloat(val); human.config.face.emotion.minConfidence = parseFloat(val);
human.config.hand.minConfidence = parseFloat(val); human.config.hand.minConfidence = parseFloat(val);
}); });
menu.process.addRange('score threshold', human.config.face.detector, 'scoreThreshold', 0.1, 1.0, 0.05, (val) => {
human.config.face.detector.scoreThreshold = parseFloat(val);
human.config.hand.scoreThreshold = parseFloat(val);
human.config.body.scoreThreshold = parseFloat(val);
});
menu.process.addRange('overlap', human.config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => { menu.process.addRange('overlap', human.config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => {
human.config.face.detector.iouThreshold = parseFloat(val); human.config.face.detector.iouThreshold = parseFloat(val);
human.config.hand.iouThreshold = parseFloat(val); human.config.hand.iouThreshold = parseFloat(val);

16
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

520
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

520
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

17
dist/human.node-gpu.js vendored
View File

File diff suppressed because one or more lines are too long

7
dist/human.node-gpu.js.map vendored
View File

File diff suppressed because one or more lines are too long

17
dist/human.node-wasm.js vendored
View File

File diff suppressed because one or more lines are too long

7
dist/human.node-wasm.js.map vendored
View File

File diff suppressed because one or more lines are too long

17
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

7
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@vladmandic/human", "name": "@vladmandic/human",
"version": "1.7.1", "version": "1.8.0",
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition", "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
"sideEffects": false, "sideEffects": false,
"main": "dist/human.node.js", "main": "dist/human.node.js",

13
server/build.js
View File

@ -49,7 +49,6 @@ const config = {
minifyWhitespace: false, minifyWhitespace: false,
minifyIdentifiers: false, minifyIdentifiers: false,
minifySyntax: false, minifySyntax: false,
sourcemap: true,
bundle: true, bundle: true,
metafile: true, metafile: true,
target: 'es2018', target: 'es2018',
@ -58,7 +57,6 @@ const config = {
minifyWhitespace: true, minifyWhitespace: true,
minifyIdentifiers: true, minifyIdentifiers: true,
minifySyntax: true, minifySyntax: true,
sourcemap: true,
bundle: true, bundle: true,
metafile: true, metafile: true,
target: 'es2018', target: 'es2018',
@ -73,6 +71,7 @@ const targets = {
entryPoints: ['src/tfjs/tf-node.ts'], entryPoints: ['src/tfjs/tf-node.ts'],
outfile: 'dist/tfjs.esm.js', outfile: 'dist/tfjs.esm.js',
external: ['@tensorflow'], external: ['@tensorflow'],
sourcemap: false,
}, },
node: { node: {
platform: 'node', platform: 'node',
@ -80,6 +79,7 @@ const targets = {
entryPoints: ['src/human.ts'], entryPoints: ['src/human.ts'],
outfile: 'dist/human.node.js', outfile: 'dist/human.node.js',
external: ['@tensorflow'], external: ['@tensorflow'],
sourcemap: false,
}, },
}, },
nodeGPU: { nodeGPU: {
@ -89,6 +89,7 @@ const targets = {
entryPoints: ['src/tfjs/tf-node-gpu.ts'], entryPoints: ['src/tfjs/tf-node-gpu.ts'],
outfile: 'dist/tfjs.esm.js', outfile: 'dist/tfjs.esm.js',
external: ['@tensorflow'], external: ['@tensorflow'],
sourcemap: false,
}, },
node: { node: {
platform: 'node', platform: 'node',
@ -96,6 +97,7 @@ const targets = {
entryPoints: ['src/human.ts'], entryPoints: ['src/human.ts'],
outfile: 'dist/human.node-gpu.js', outfile: 'dist/human.node-gpu.js',
external: ['@tensorflow'], external: ['@tensorflow'],
sourcemap: false,
}, },
}, },
nodeWASM: { nodeWASM: {
@ -105,6 +107,7 @@ const targets = {
entryPoints: ['src/tfjs/tf-node-wasm.ts'], entryPoints: ['src/tfjs/tf-node-wasm.ts'],
outfile: 'dist/tfjs.esm.js', outfile: 'dist/tfjs.esm.js',
external: ['@tensorflow'], external: ['@tensorflow'],
sourcemap: false,
}, },
node: { node: {
platform: 'node', platform: 'node',
@ -112,6 +115,7 @@ const targets = {
entryPoints: ['src/human.ts'], entryPoints: ['src/human.ts'],
outfile: 'dist/human.node-wasm.js', outfile: 'dist/human.node-wasm.js',
external: ['@tensorflow'], external: ['@tensorflow'],
sourcemap: false,
}, },
}, },
@ -122,6 +126,7 @@ const targets = {
entryPoints: ['src/tfjs/tf-browser.ts'], entryPoints: ['src/tfjs/tf-browser.ts'],
outfile: 'dist/tfjs.esm.js', outfile: 'dist/tfjs.esm.js',
external: ['fs', 'buffer', 'util', 'os', '@tensorflow'], external: ['fs', 'buffer', 'util', 'os', '@tensorflow'],
sourcemap: true,
}, },
esm: { esm: {
platform: 'browser', platform: 'browser',
@ -129,6 +134,7 @@ const targets = {
entryPoints: ['src/human.ts'], entryPoints: ['src/human.ts'],
outfile: 'dist/human.esm-nobundle.js', outfile: 'dist/human.esm-nobundle.js',
external: ['fs', 'buffer', 'util', 'os', '@tensorflow'], external: ['fs', 'buffer', 'util', 'os', '@tensorflow'],
sourcemap: true,
}, },
}, },
browserBundle: { browserBundle: {
@ -138,6 +144,7 @@ const targets = {
entryPoints: ['src/tfjs/tf-browser.ts'], entryPoints: ['src/tfjs/tf-browser.ts'],
outfile: 'dist/tfjs.esm.js', outfile: 'dist/tfjs.esm.js',
external: ['fs', 'buffer', 'util', 'os'], external: ['fs', 'buffer', 'util', 'os'],
sourcemap: true,
}, },
iife: { iife: {
platform: 'browser', platform: 'browser',
@ -146,6 +153,7 @@ const targets = {
entryPoints: ['src/human.ts'], entryPoints: ['src/human.ts'],
outfile: 'dist/human.js', outfile: 'dist/human.js',
external: ['fs', 'buffer', 'util', 'os'], external: ['fs', 'buffer', 'util', 'os'],
sourcemap: true,
}, },
esm: { esm: {
platform: 'browser', platform: 'browser',
@ -153,6 +161,7 @@ const targets = {
entryPoints: ['src/human.ts'], entryPoints: ['src/human.ts'],
outfile: 'dist/human.esm.js', outfile: 'dist/human.esm.js',
external: ['fs', 'buffer', 'util', 'os'], external: ['fs', 'buffer', 'util', 'os'],
sourcemap: true,
}, },
/* /*
demo: { demo: {

34
server/build.log
View File

@ -236,3 +236,37 @@
2021-04-25 07:50:17 INFO:  Generate types: ["src/human.ts"] 2021-04-25 07:50:17 INFO:  Generate types: ["src/human.ts"]
2021-04-25 07:50:22 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"] 2021-04-25 07:50:22 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-25 07:50:22 INFO:  Generate TypeDocs: ["src/human.ts"] 2021-04-25 07:50:22 INFO:  Generate TypeDocs: ["src/human.ts"]
2021-04-25 13:08:21 INFO:  @vladmandic/human version 1.7.1
2021-04-25 13:08:21 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-04-25 13:08:21 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true,"sourcemap":true,"bundle":true,"metafile":true,"target":"es2018"}
2021-04-25 13:08:21 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":733,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:08:21 STATE: Build for: node type: node: {"imports":35,"importBytes":514895,"outputBytes":294872,"outputFiles":"dist/human.node.js"}
2021-04-25 13:08:21 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":737,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:08:21 STATE: Build for: nodeGPU type: node: {"imports":35,"importBytes":514899,"outputBytes":294880,"outputFiles":"dist/human.node-gpu.js"}
2021-04-25 13:08:21 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":783,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:08:21 STATE: Build for: nodeWASM type: node: {"imports":35,"importBytes":514945,"outputBytes":294924,"outputFiles":"dist/human.node-wasm.js"}
2021-04-25 13:08:21 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:08:21 STATE: Build for: browserNoBundle type: esm: {"imports":35,"importBytes":515556,"outputBytes":295030,"outputFiles":"dist/human.esm-nobundle.js"}
2021-04-25 13:08:22 STATE: Build for: browserBundle type: tfjs: {"modules":1267,"moduleBytes":4085087,"imports":7,"importBytes":2488,"outputBytes":1101728,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:08:23 STATE: Build for: browserBundle type: iife: {"imports":35,"importBytes":1615890,"outputBytes":1393163,"outputFiles":"dist/human.js"}
2021-04-25 13:08:23 STATE: Build for: browserBundle type: esm: {"imports":35,"importBytes":1615890,"outputBytes":1393121,"outputFiles":"dist/human.esm.js"}
2021-04-25 13:08:23 INFO:  Generate types: ["src/human.ts"]
2021-04-25 13:08:28 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-25 13:08:28 INFO:  Generate TypeDocs: ["src/human.ts"]
2021-04-25 13:13:05 INFO:  @vladmandic/human version 1.8.0
2021-04-25 13:13:05 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-04-25 13:13:05 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true,"bundle":true,"metafile":true,"target":"es2018"}
2021-04-25 13:13:05 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":696,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:13:05 STATE: Build for: node type: node: {"imports":35,"importBytes":514858,"outputBytes":294833,"outputFiles":"dist/human.node.js"}
2021-04-25 13:13:05 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":700,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:13:05 STATE: Build for: nodeGPU type: node: {"imports":35,"importBytes":514862,"outputBytes":294837,"outputFiles":"dist/human.node-gpu.js"}
2021-04-25 13:13:05 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":746,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:13:06 STATE: Build for: nodeWASM type: node: {"imports":35,"importBytes":514908,"outputBytes":294880,"outputFiles":"dist/human.node-wasm.js"}
2021-04-25 13:13:06 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:13:06 STATE: Build for: browserNoBundle type: esm: {"imports":35,"importBytes":515556,"outputBytes":295030,"outputFiles":"dist/human.esm-nobundle.js"}
2021-04-25 13:13:06 STATE: Build for: browserBundle type: tfjs: {"modules":1267,"moduleBytes":4085087,"imports":7,"importBytes":2488,"outputBytes":1101728,"outputFiles":"dist/tfjs.esm.js"}
2021-04-25 13:13:07 STATE: Build for: browserBundle type: iife: {"imports":35,"importBytes":1615890,"outputBytes":1393163,"outputFiles":"dist/human.js"}
2021-04-25 13:13:08 STATE: Build for: browserBundle type: esm: {"imports":35,"importBytes":1615890,"outputBytes":1393121,"outputFiles":"dist/human.esm.js"}
2021-04-25 13:13:08 INFO:  Generate types: ["src/human.ts"]
2021-04-25 13:13:12 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-25 13:13:12 INFO:  Generate TypeDocs: ["src/human.ts"]

10
src/age/age.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers'; import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
let model; let model;
let last = { age: 0 }; let last = { age: 0 };
@ -31,14 +30,7 @@ export async function predict(image, config) {
let ageT; let ageT;
const obj = { age: 0 }; const obj = { age: 0 };
if (!config.profile) { if (config.face.age.enabled) ageT = await model.predict(enhance);
if (config.face.age.enabled) ageT = await model.predict(enhance);
} else {
const profileAge = config.face.age.enabled ? await tf.profile(() => model.predict(enhance)) : {};
ageT = profileAge.result.clone();
profileAge.result.dispose();
profile.run('age', profileAge);
}
enhance.dispose(); enhance.dispose();
if (ageT) { if (ageT) {

2
src/blazeface/blazeface.ts
View File

@ -90,7 +90,7 @@ export class BlazeFaceModel {
const scoresOut = tf.sigmoid(logits).squeeze(); const scoresOut = tf.sigmoid(logits).squeeze();
return [batchOut, boxesOut, scoresOut]; return [batchOut, boxesOut, scoresOut];
}); });
const boxIndicesTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxFaces, this.config.face.detector.iouThreshold, this.config.face.detector.scoreThreshold); const boxIndicesTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
const boxIndices = boxIndicesTensor.arraySync(); const boxIndices = boxIndicesTensor.arraySync();
boxIndicesTensor.dispose(); boxIndicesTensor.dispose();
const boundingBoxesMap = boxIndices.map((boxIndex) => tf.slice(boxes, [boxIndex, 0], [1, -1])); const boundingBoxesMap = boxIndices.map((boxIndex) => tf.slice(boxes, [boxIndex, 0], [1, -1]));

2
src/blazeface/facepipeline.ts
View File

@ -161,7 +161,7 @@ export class Pipeline {
if (config.videoOptimized) this.skipped++; if (config.videoOptimized) this.skipped++;
// if detector result count doesn't match current working set, use it to reset current working set // if detector result count doesn't match current working set, use it to reset current working set
if (!config.videoOptimized || (detector && detector.boxes && (!config.face.mesh.enabled || (detector.boxes.length !== this.detectedFaces) && (this.detectedFaces !== config.face.detector.maxFaces)))) { if (!config.videoOptimized || (detector && detector.boxes && (!config.face.mesh.enabled || (detector.boxes.length !== this.detectedFaces) && (this.detectedFaces !== config.face.detector.maxDetected)))) {
this.storedBoxes = []; this.storedBoxes = [];
this.detectedFaces = 0; this.detectedFaces = 0;
for (const possible of detector.boxes) { for (const possible of detector.boxes) {

15
src/blazepose/blazepose.ts
View File

@ -2,7 +2,6 @@
import { log, join } from '../helpers'; import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
import * as annotations from './annotations'; import * as annotations from './annotations';
let model; let model;
@ -25,17 +24,9 @@ export async function predict(image, config) {
const resize = tf.image.resizeBilinear(image, [model.width, model.height], false); const resize = tf.image.resizeBilinear(image, [model.width, model.height], false);
const normalize = tf.div(resize, [255.0]); const normalize = tf.div(resize, [255.0]);
resize.dispose(); resize.dispose();
let points; const resT = await model.predict(normalize);
if (!config.profile) { // run through profiler or just execute const points = resT.find((t) => (t.size === 195 || t.size === 155)).dataSync(); // order of output tensors may change between models, full has 195 and upper has 155 items
const resT = await model.predict(normalize); resT.forEach((t) => t.dispose());
points = resT.find((t) => (t.size === 195 || t.size === 155)).dataSync(); // order of output tensors may change between models, full has 195 and upper has 155 items
resT.forEach((t) => t.dispose());
} else {
const profileData = await tf.profile(() => model.predict(normalize));
points = profileData.result.find((t) => (t.size === 195 || t.size === 155)).dataSync();
profileData.result.forEach((t) => t.dispose());
profile.run('blazepose', profileData);
}
normalize.dispose(); normalize.dispose();
const keypoints: Array<{ id, part, position: { x, y, z }, score, presence }> = []; const keypoints: Array<{ id, part, position: { x, y, z }, score, presence }> = [];
const labels = points.length === 195 ? annotations.full : annotations.upper; // full model has 39 keypoints, upper has 31 keypoints const labels = points.length === 195 ? annotations.full : annotations.upper; // full model has 39 keypoints, upper has 31 keypoints

148
src/config.ts
View File

@ -9,33 +9,34 @@
export interface Config { export interface Config {
/** Backend used for TFJS operations */ /** Backend used for TFJS operations */
backend: null | '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow', backend: null | '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow',
/** Path to *.wasm files if backend is set to `wasm` */ /** Path to *.wasm files if backend is set to `wasm` */
wasmPath: string, wasmPath: string,
/** Print debug statements to console */ /** Print debug statements to console */
debug: boolean, debug: boolean,
/** Perform model loading and inference concurrently or sequentially */ /** Perform model loading and inference concurrently or sequentially */
async: boolean, async: boolean,
/** Collect and print profiling data during inference operations */
profile: boolean,
/** Internal: Use aggressive GPU memory deallocator when backend is set to `webgl` or `humangl` */
deallocate: boolean,
/** Internal: Run all inference operations in an explicit local scope run to avoid memory leaks */
scoped: boolean,
/** Perform additional optimizations when input is video, /** Perform additional optimizations when input is video,
* - must be disabled for images * - must be disabled for images
* - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs * - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs
* - skips boundary detection for every `skipFrames` frames specified for each model * - skips boundary detection for every `skipFrames` frames specified for each model
* - while maintaining in-box detection since objects don't change definition as fast */ * - while maintaining in-box detection since objects don't change definition as fast */
videoOptimized: boolean, videoOptimized: boolean,
/** What to use for `human.warmup()` /** What to use for `human.warmup()`
* - warmup pre-initializes all models for faster inference but can take significant time on startup * - warmup pre-initializes all models for faster inference but can take significant time on startup
* - only used for `webgl` and `humangl` backends * - only used for `webgl` and `humangl` backends
*/ */
warmup: 'none' | 'face' | 'full' | 'body', warmup: 'none' | 'face' | 'full' | 'body',
/** Base model path (typically starting with file://, http:// or https://) for all models /** Base model path (typically starting with file://, http:// or https://) for all models
* - individual modelPath values are joined to this path * - individual modelPath values are relative to this path
*/ */
modelBasePath: string, modelBasePath: string,
/** Run input through image filters before inference /** Run input through image filters before inference
* - image filters run with near-zero latency as they are executed on the GPU * - image filters run with near-zero latency as they are executed on the GPU
*/ */
@ -90,31 +91,30 @@ export interface Config {
gesture: { gesture: {
enabled: boolean, enabled: boolean,
}, },
/** Controlls and configures all face-specific options: /** Controlls and configures all face-specific options:
* - face detection, face mesh detection, age, gender, emotion detection and face description * - face detection, face mesh detection, age, gender, emotion detection and face description
* Parameters: * Parameters:
* - enabled: true/false * - enabled: true/false
* - modelPath: path for individual face model * - modelPath: path for each of face models
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance
* - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles * - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles
* - maxFaces: maximum number of faces detected in the input, should be set to the minimum number for performance
* - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true * - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true
* - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle * - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle
* - minConfidence: threshold for discarding a prediction * - return: return extracted face as tensor for futher user processing
* - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
* - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
* - return extracted face as tensor for futher user processing
*/ */
face: { face: {
enabled: boolean, enabled: boolean,
detector: { detector: {
modelPath: string, modelPath: string,
rotation: boolean, rotation: boolean,
maxFaces: number, maxDetected: number,
skipFrames: number, skipFrames: number,
skipInitial: boolean, skipInitial: boolean,
minConfidence: number, minConfidence: number,
iouThreshold: number, iouThreshold: number,
scoreThreshold: number,
return: boolean, return: boolean,
}, },
mesh: { mesh: {
@ -138,31 +138,30 @@ export interface Config {
modelPath: string, modelPath: string,
}, },
}, },
/** Controlls and configures all body detection specific options /** Controlls and configures all body detection specific options
* - enabled: true/false * - enabled: true/false
* - modelPath: paths for both hand detector model and hand skeleton model * - modelPath: body pose model, can be absolute path or relative to modelBasePath
* - maxDetections: maximum number of people detected in the input, should be set to the minimum number for performance * - minConfidence: threshold for discarding a prediction
* - scoreThreshold: threshold for deciding when to remove people based on score in non-maximum suppression * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance
* - nmsRadius: threshold for deciding whether body parts overlap too much in non-maximum suppression
*/ */
body: { body: {
enabled: boolean, enabled: boolean,
modelPath: string, modelPath: string,
maxDetections: number, maxDetected: number,
scoreThreshold: number, minConfidence: number,
nmsRadius: number,
}, },
/** Controlls and configures all hand detection specific options /** Controlls and configures all hand detection specific options
* - enabled: true/false * - enabled: true/false
* - modelPath: paths for both hand detector model and hand skeleton model * - landmarks: detect hand landmarks or just hand boundary box
* - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance
* - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
* - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true * - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true
* - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle * - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
* - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
* - maxHands: maximum number of hands detected in the input, should be set to the minimum number for performance
* - landmarks: detect hand landmarks or just hand boundary box
*/ */
hand: { hand: {
enabled: boolean, enabled: boolean,
@ -171,8 +170,7 @@ export interface Config {
skipInitial: boolean, skipInitial: boolean,
minConfidence: number, minConfidence: number,
iouThreshold: number, iouThreshold: number,
scoreThreshold: number, maxDetected: number,
maxHands: number,
landmarks: boolean, landmarks: boolean,
detector: { detector: {
modelPath: string, modelPath: string,
@ -181,10 +179,13 @@ export interface Config {
modelPath: string, modelPath: string,
}, },
}, },
/** Controlls and configures all object detection specific options /** Controlls and configures all object detection specific options
* - enabled: true/false
* - modelPath: object detection model, can be absolute path or relative to modelBasePath
* - minConfidence: minimum score that detection must have to return as valid object * - minConfidence: minimum score that detection must have to return as valid object
* - iouThreshold: ammount of overlap between two detected objects before one object is removed * - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxResults: maximum number of detections to return * - maxDetected: maximum number of detections to return
* - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true * - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true
*/ */
object: { object: {
@ -192,40 +193,20 @@ export interface Config {
modelPath: string, modelPath: string,
minConfidence: number, minConfidence: number,
iouThreshold: number, iouThreshold: number,
maxResults: number, maxDetected: number,
skipFrames: number, skipFrames: number,
}, },
} }
const config: Config = { const config: Config = {
backend: 'webgl', // select tfjs backend to use backend: 'webgl', // select tfjs backend to use, leave empty to use default backend
// can be 'webgl', 'wasm', 'cpu', or 'humangl' which is a custom version of webgl // can be 'webgl', 'wasm', 'cpu', or 'humangl' which is a custom version of webgl
// leave as empty string to continue using default backend
// when backend is set outside of Human library
modelBasePath: '../models/', // base path for all models modelBasePath: '../models/', // base path for all models
wasmPath: '../assets/', // path for wasm binaries wasmPath: '../assets/', // path for wasm binariesm, only used for backend: wasm
// only used for backend: wasm
debug: true, // print additional status messages to console debug: true, // print additional status messages to console
async: true, // execute enabled models in parallel async: true, // execute enabled models in parallel
// this disables per-model performance data but
// slightly increases performance
// cannot be used if profiling is enabled
profile: false, // internal: enable tfjs profiling
// this has significant performance impact
// only enable for debugging purposes
// currently only implemented for age,gender,emotion models
deallocate: false, // internal: aggresively deallocate gpu memory after each usage
// only valid for webgl and humangl backend and only during first call
// cannot be changed unless library is reloaded
// this has significant performance impact
// only enable on low-memory devices
scoped: false, // internal: enable scoped runs
// some models *may* have memory leaks,
// this wrapps everything in a local scope at a cost of performance
// typically not needed
videoOptimized: true, // perform additional optimizations when input is video, videoOptimized: true, // perform additional optimizations when input is video,
// must be disabled for images // automatically disabled for Image, ImageData, ImageBitmap
// automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs
// skips boundary detection for every n frames // skips boundary detection for every n frames
// while maintaining in-box detection since objects cannot move that fast // while maintaining in-box detection since objects cannot move that fast
warmup: 'face', // what to use for human.warmup(), can be 'none', 'face', 'full' warmup: 'face', // what to use for human.warmup(), can be 'none', 'face', 'full'
@ -258,7 +239,7 @@ const config: Config = {
}, },
gesture: { gesture: {
enabled: true, // enable simple gesture recognition enabled: true, // enable gesture recognition based on model results
}, },
face: { face: {
@ -267,12 +248,11 @@ const config: Config = {
// detector, mesh, iris, age, gender, emotion // detector, mesh, iris, age, gender, emotion
// (note: module is not loaded until it is required) // (note: module is not loaded until it is required)
detector: { detector: {
modelPath: 'blazeface-back.json', // detector model modelPath: 'blazeface-back.json', // detector model, can be absolute path or relative to modelBasePath
// can be either absolute path or relative to modelBasePath
rotation: false, // use best-guess rotated face image or just box with rotation as-is rotation: false, // use best-guess rotated face image or just box with rotation as-is
// false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees // false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees
// this parameter is not valid in nodejs // this parameter is not valid in nodejs
maxFaces: 10, // maximum number of faces detected in the input maxDetected: 10, // maximum number of faces detected in the input
// should be set to the minimum number for performance // should be set to the minimum number for performance
skipFrames: 21, // how many frames to go without re-running the face bounding box detector skipFrames: 21, // how many frames to go without re-running the face bounding box detector
// only used for video inputs // only used for video inputs
@ -282,18 +262,13 @@ const config: Config = {
skipInitial: false, // if previous detection resulted in no faces detected, skipInitial: false, // if previous detection resulted in no faces detected,
// should skipFrames be reset immediately to force new detection cycle // should skipFrames be reset immediately to force new detection cycle
minConfidence: 0.2, // threshold for discarding a prediction minConfidence: 0.2, // threshold for discarding a prediction
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed
// non-maximum suppression (0.1 means drop if overlap 10%)
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score
// in non-maximum suppression,
// this is applied on detection objects only and before minConfidence
return: false, // return extracted face as tensor return: false, // return extracted face as tensor
}, },
mesh: { mesh: {
enabled: true, enabled: true,
modelPath: 'facemesh.json', // facemesh model modelPath: 'facemesh.json', // facemesh model, can be absolute path or relative to modelBasePath
// can be either absolute path or relative to modelBasePath
}, },
iris: { iris: {
@ -316,25 +291,18 @@ const config: Config = {
enabled: true, enabled: true,
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
skipFrames: 32, // how many frames to go without re-running the detector skipFrames: 32, // how many frames to go without re-running the detector
modelPath: 'emotion.json', // face emotion model modelPath: 'emotion.json', // face emotion model, can be absolute path or relative to modelBasePath
// can be either absolute path or relative to modelBasePath
}, },
}, },
body: { body: {
enabled: true, enabled: true,
modelPath: 'posenet.json', // body model modelPath: 'posenet.json', // body model, can be absolute path or relative to modelBasePath
// can be either absolute path or relative to modelBasePath // can be 'posenet' or 'blazepose'
// can be 'posenet', 'blazepose' or 'efficientpose' maxDetected: 1, // maximum number of people detected in the input
// 'blazepose' and 'efficientpose' are experimental
maxDetections: 1, // maximum number of people detected in the input
// should be set to the minimum number for performance // should be set to the minimum number for performance
// only valid for posenet as blazepose only detects single pose // only valid for posenet as blazepose only detects single pose
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score minConfidence: 0.2, // threshold for discarding a prediction
// in non-maximum suppression
// only valid for posenet as blazepose only detects single pose
nmsRadius: 20, // radius for deciding points are too close in non-maximum suppression
// only valid for posenet as blazepose only detects single pose
}, },
hand: { hand: {
@ -349,32 +317,24 @@ const config: Config = {
skipInitial: false, // if previous detection resulted in no hands detected, skipInitial: false, // if previous detection resulted in no hands detected,
// should skipFrames be reset immediately to force new detection cycle // should skipFrames be reset immediately to force new detection cycle
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed
// in non-maximum suppression maxDetected: 1, // maximum number of hands detected in the input
scoreThreshold: 0.5, // threshold for deciding when to remove boxes based on
// score in non-maximum suppression
maxHands: 1, // maximum number of hands detected in the input
// should be set to the minimum number for performance // should be set to the minimum number for performance
landmarks: true, // detect hand landmarks or just hand boundary box landmarks: true, // detect hand landmarks or just hand boundary box
detector: { detector: {
modelPath: 'handdetect.json', // hand detector model modelPath: 'handdetect.json', // hand detector model, can be absolute path or relative to modelBasePath
// can be either absolute path or relative to modelBasePath
}, },
skeleton: { skeleton: {
modelPath: 'handskeleton.json', // hand skeleton model modelPath: 'handskeleton.json', // hand skeleton model, can be absolute path or relative to modelBasePath
// can be either absolute path or relative to modelBasePath
}, },
}, },
object: { object: {
enabled: false, enabled: false,
modelPath: 'nanodet.json', // object detection model modelPath: 'nanodet.json', // experimental: object detection model, can be absolute path or relative to modelBasePath
// can be either absolute path or relative to modelBasePath minConfidence: 0.2, // threshold for discarding a prediction
// 'nanodet' is experimental iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed
minConfidence: 0.20, // threshold for discarding a prediction maxDetected: 10, // maximum number of objects detected in the input
iouThreshold: 0.40, // threshold for deciding whether boxes overlap too much
// in non-maximum suppression
maxResults: 10, // maximum number of objects detected in the input
skipFrames: 41, // how many frames to go without re-running the detector skipFrames: 41, // how many frames to go without re-running the detector
}, },
}; };

48
src/draw/draw.ts
View File

@ -294,68 +294,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, draw
// shoulder line // shoulder line
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// torso main // torso main
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHip'); part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHip'); part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
// leg left // leg left
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftHip'); part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftKnee'); part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHeel'); part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftFoot'); part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// leg right // leg right
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightHip'); part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightKnee'); part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHeel'); part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightFoot'); part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// arm left // arm left
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftElbow'); part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftWrist'); part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftPalm'); part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// arm right // arm right
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightElbow'); part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightWrist'); part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightPalm'); part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// draw all // draw all
} }

15
src/efficientpose/efficientpose.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers'; import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
let model; let model;
let keypoints: Array<any> = []; let keypoints: Array<any> = [];
@ -55,15 +54,7 @@ export async function predict(image, config) {
}); });
let resT; let resT;
if (config.body.enabled) resT = await model.predict(tensor);
if (!config.profile) {
if (config.body.enabled) resT = await model.predict(tensor);
} else {
const profileT = config.body.enabled ? await tf.profile(() => model.predict(tensor)) : {};
resT = profileT.result.clone();
profileT.result.dispose();
profile.run('body', profileT);
}
tensor.dispose(); tensor.dispose();
if (resT) { if (resT) {
@ -76,8 +67,8 @@ export async function predict(image, config) {
// process each unstacked tensor as a separate body part // process each unstacked tensor as a separate body part
for (let id = 0; id < stack.length; id++) { for (let id = 0; id < stack.length; id++) {
// actual processing to get coordinates and score // actual processing to get coordinates and score
const [x, y, score] = max2d(stack[id], config.body.scoreThreshold); const [x, y, score] = max2d(stack[id], config.body.minConfidence);
if (score > config.body.scoreThreshold) { if (score > config.body.minConfidence) {
parts.push({ parts.push({
id, id,
score: Math.round(100 * score) / 100, score: Math.round(100 * score) / 100,

66
src/embedding/embedding.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers'; import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
type Tensor = typeof tf.Tensor; type Tensor = typeof tf.Tensor;
type DB = Array<{ name: string, source: string, embedding: number[] }>; type DB = Array<{ name: string, source: string, embedding: number[] }>;
@ -87,46 +86,39 @@ export async function predict(input, config): Promise<number[]> {
let data: Array<number> = []; let data: Array<number> = [];
if (config.face.embedding.enabled) { if (config.face.embedding.enabled) {
const image = enhance(input); const image = enhance(input);
if (!config.profile) { data = tf.tidy(() => {
data = tf.tidy(() => { /*
/* // if needed convert from NHWC to NCHW
// if needed convert from NHWC to NCHW const nchw = image.transpose([3, 0, 1, 2]);
const nchw = image.transpose([3, 0, 1, 2]); */
*/
const res = model.predict(image); const res = model.predict(image);
/* /*
// optionally do it twice with flipped image and average results // optionally do it twice with flipped image and average results
const res1 = model.predict(image); const res1 = model.predict(image);
const flipped = tf.image.flipLeftRight(image); const flipped = tf.image.flipLeftRight(image);
const res2 = model.predict(flipped); const res2 = model.predict(flipped);
const merge = tf.stack([res1, res2], 2).squeeze(); const merge = tf.stack([res1, res2], 2).squeeze();
const res = reshape.logSumExp(1); const res = reshape.logSumExp(1);
*/ */
/* /*
// optional normalize outputs with l2 normalization // optional normalize outputs with l2 normalization
const scaled = tf.tidy(() => { const scaled = tf.tidy(() => {
const l2 = res.norm('euclidean'); const l2 = res.norm('euclidean');
const scale = res.div(l2); const scale = res.div(l2);
return scale; return scale;
});
*/
// optional reduce feature vector complexity
const reshape = res.reshape([128, 2]); // split 256 vectors into 128 x 2
const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it
const output: Array<number> = reduce.dataSync();
return [...output]; // convert typed array to simple array
}); });
} else { */
const profileData = await tf.profile(() => model.predict({ img_inputs: image }));
data = [...profileData.result.dataSync()]; // optional reduce feature vector complexity
profileData.result.dispose(); const reshape = res.reshape([128, 2]); // split 256 vectors into 128 x 2
profile.run('emotion', profileData); const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it
}
const output: Array<number> = reduce.dataSync();
return [...output]; // convert typed array to simple array
});
tf.dispose(image); tf.dispose(image);
} }
resolve(data); resolve(data);

15
src/emotion/emotion.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers'; import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']; const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'];
let model; let model;
@ -46,17 +45,9 @@ export async function predict(image, config) {
grayscale.dispose(); grayscale.dispose();
const obj: Array<{ score: number, emotion: string }> = []; const obj: Array<{ score: number, emotion: string }> = [];
if (config.face.emotion.enabled) { if (config.face.emotion.enabled) {
let data; const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation
if (!config.profile) { const data = emotionT.dataSync();
const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation tf.dispose(emotionT);
data = emotionT.dataSync();
tf.dispose(emotionT);
} else {
const profileData = await tf.profile(() => model.predict(normalize));
data = profileData.result.dataSync();
profileData.result.dispose();
profile.run('emotion', profileData);
}
for (let i = 0; i < data.length; i++) { for (let i = 0; i < data.length; i++) {
if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] }); if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
} }

9
src/faceres/faceres.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers'; import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
let model; let model;
let last = { age: 0 }; let last = { age: 0 };
@ -108,13 +107,7 @@ export async function predict(image, config) {
genderConfidence: <number>0, genderConfidence: <number>0,
descriptor: <number[]>[] }; descriptor: <number[]>[] };
if (!config.profile) { if (config.face.description.enabled) resT = await model.predict(enhanced);
if (config.face.description.enabled) resT = await model.predict(enhanced);
} else {
const profileDesc = config.face.description.enabled ? await tf.profile(() => model.predict(enhanced)) : {};
resT = profileDesc.result;
profile.run('faceres', profileDesc);
}
tf.dispose(enhanced); tf.dispose(enhanced);
if (resT) { if (resT) {

10
src/gender/gender.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers'; import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
let model; let model;
let last = { gender: '' }; let last = { gender: '' };
@ -49,14 +48,7 @@ export async function predict(image, config) {
let genderT; let genderT;
const obj = { gender: '', confidence: 0 }; const obj = { gender: '', confidence: 0 };
if (!config.profile) { if (config.face.gender.enabled) genderT = await model.predict(enhance);
if (config.face.gender.enabled) genderT = await model.predict(enhance);
} else {
const profileGender = config.face.gender.enabled ? await tf.profile(() => model.predict(enhance)) : {};
genderT = profileGender.result.clone();
profileGender.result.dispose();
profile.run('gender', profileGender);
}
enhance.dispose(); enhance.dispose();
if (genderT) { if (genderT) {

2
src/handpose/handdetector.ts
View File

@ -46,7 +46,7 @@ export class HandDetector {
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]); const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); const boxes = this.normalizeBoxes(rawBoxes);
rawBoxes.dispose(); rawBoxes.dispose();
const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxHands, config.hand.iouThreshold, config.hand.scoreThreshold); const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
const filtered = filteredT.arraySync(); const filtered = filteredT.arraySync();
scoresT.dispose(); scoresT.dispose();

2
src/handpose/handpipeline.ts
View File

@ -83,7 +83,7 @@ export class HandPipeline {
if (config.videoOptimized) this.skipped++; if (config.videoOptimized) this.skipped++;
// if detector result count doesn't match current working set, use it to reset current working set // if detector result count doesn't match current working set, use it to reset current working set
if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxHands) || !config.hand.landmarks)) { if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxDetected) || !config.hand.landmarks)) {
this.detectedHands = 0; this.detectedHands = 0;
this.storedBoxes = [...boxes]; this.storedBoxes = [...boxes];
// for (const possible of boxes) this.storedBoxes.push(possible); // for (const possible of boxes) this.storedBoxes.push(possible);

21
src/human.ts
View File

@ -13,7 +13,6 @@ import * as nanodet from './nanodet/nanodet';
import * as gesture from './gesture/gesture'; import * as gesture from './gesture/gesture';
import * as image from './image/image'; import * as image from './image/image';
import * as draw from './draw/draw'; import * as draw from './draw/draw';
import * as profile from './profile';
import { Config, defaults } from './config'; import { Config, defaults } from './config';
import { Result } from './result'; import { Result } from './result';
import * as sample from './sample'; import * as sample from './sample';
@ -168,14 +167,6 @@ export class Human {
this.sysinfo = sysinfo.info(); this.sysinfo = sysinfo.info();
} }
/** Internal: ProfileData method returns last known profiling information
* - Requires human.config.profile set to true
*/
profileData(): { newBytes, newTensors, peakBytes, numKernelOps, timeKernelOps, slowestKernelOps, largestKernelOps } | {} {
if (this.config.profile) return profile.data;
return {};
}
// helper function: measure tensor leak // helper function: measure tensor leak
/** @hidden */ /** @hidden */
analyze = (...msg) => { analyze = (...msg) => {
@ -335,9 +326,9 @@ export class Human {
if (this.tf.getBackend() === 'webgl' || this.tf.getBackend() === 'humangl') { if (this.tf.getBackend() === 'webgl' || this.tf.getBackend() === 'humangl') {
this.tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false); this.tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false);
this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true); this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
if (this.config.deallocate) { if (typeof this.config['deallocate'] !== 'undefined') {
log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate); log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true);
this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1); this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', 0);
} }
const gl = await this.tf.backend().getGPGPUContext().gl; const gl = await this.tf.backend().getGPGPUContext().gl;
if (this.config.debug) log(`gl version:${gl.getParameter(gl.VERSION)} renderer:${gl.getParameter(gl.RENDERER)}`); if (this.config.debug) log(`gl version:${gl.getParameter(gl.VERSION)} renderer:${gl.getParameter(gl.RENDERER)}`);
@ -378,9 +369,6 @@ export class Human {
// load models if enabled // load models if enabled
await this.load(); await this.load();
if (this.config.scoped) this.tf.engine().startScope();
this.analyze('Start Scope:');
// disable video optimization for inputs of type image, but skip if inside worker thread // disable video optimization for inputs of type image, but skip if inside worker thread
let previousVideoOptimized; let previousVideoOptimized;
// @ts-ignore ignore missing type for WorkerGlobalScope as that is the point // @ts-ignore ignore missing type for WorkerGlobalScope as that is the point
@ -474,9 +462,6 @@ export class Human {
} }
tf.dispose(process.tensor); tf.dispose(process.tensor);
if (this.config.scoped) this.tf.engine().endScope();
this.analyze('End Scope:');
// run gesture analysis last // run gesture analysis last
let gestureRes: any[] = []; let gestureRes: any[] = [];
if (this.config.gesture.enabled) { if (this.config.gesture.enabled) {

11
src/nanodet/nanodet.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers'; import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
import { labels } from './labels'; import { labels } from './labels';
let model; let model;
@ -83,7 +82,7 @@ async function process(res, inputSize, outputShape, config) {
const nmsScores = results.map((a) => a.score); const nmsScores = results.map((a) => a.score);
let nmsIdx: any[] = []; let nmsIdx: any[] = [];
if (nmsBoxes && nmsBoxes.length > 0) { if (nmsBoxes && nmsBoxes.length > 0) {
const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxResults, config.object.iouThreshold, config.object.minConfidence); const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
nmsIdx = nms.dataSync(); nmsIdx = nms.dataSync();
tf.dispose(nms); tf.dispose(nms);
} }
@ -114,13 +113,7 @@ export async function predict(image, config) {
resize.dispose(); resize.dispose();
let objectT; let objectT;
if (!config.profile) { if (config.object.enabled) objectT = await model.predict(transpose);
if (config.object.enabled) objectT = await model.predict(transpose);
} else {
const profileObject = config.object.enabled ? await tf.profile(() => model.predict(transpose)) : {};
objectT = profileObject.result;
profile.run('object', profileObject);
}
transpose.dispose(); transpose.dispose();
const obj = await process(objectT, model.inputSize, outputSize, config); const obj = await process(objectT, model.inputSize, outputSize, config);

2
src/posenet/posenet.ts
View File

@ -19,7 +19,7 @@ export async function predict(input, config) {
const buffers = await Promise.all(res.map((tensor) => tensor.buffer())); const buffers = await Promise.all(res.map((tensor) => tensor.buffer()));
for (const t of res) t.dispose(); for (const t of res) t.dispose();
const decoded = await poses.decode(buffers[0], buffers[1], buffers[2], buffers[3], config.body.nmsRadius, config.body.maxDetections, config.body.scoreThreshold); const decoded = await poses.decode(buffers[0], buffers[1], buffers[2], buffers[3], config.body.maxDetected, config.body.minConfidence);
const scaled = util.scalePoses(decoded, [input.shape[1], input.shape[2]], [model.inputs[0].shape[2], model.inputs[0].shape[1]]); const scaled = util.scalePoses(decoded, [input.shape[1], input.shape[2]], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
return scaled; return scaled;

28
src/posenet/poses.ts
View File

@ -3,6 +3,7 @@ import * as kpt from './keypoints';
const localMaximumRadius = 1; const localMaximumRadius = 1;
const defaultOutputStride = 16; const defaultOutputStride = 16;
const squaredNmsRadius = 20 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) { function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
const getDisplacement = (point) => ({ const getDisplacement = (point) => ({
@ -86,7 +87,7 @@ function scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scor
return localMaximum; return localMaximum;
} }
export function buildPartWithScoreQueue(scoreThreshold, scores) { export function buildPartWithScoreQueue(minConfidence, scores) {
const [height, width, numKeypoints] = scores.shape; const [height, width, numKeypoints] = scores.shape;
const queue = new utils.MaxHeap(height * width * numKeypoints, ({ score }) => score); const queue = new utils.MaxHeap(height * width * numKeypoints, ({ score }) => score);
for (let heatmapY = 0; heatmapY < height; ++heatmapY) { for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
@ -94,7 +95,7 @@ export function buildPartWithScoreQueue(scoreThreshold, scores) {
for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) { for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
const score = scores.get(heatmapY, heatmapX, keypointId); const score = scores.get(heatmapY, heatmapX, keypointId);
// Only consider parts with score greater or equal to threshold as root candidates. // Only consider parts with score greater or equal to threshold as root candidates.
if (score < scoreThreshold) continue; if (score < minConfidence) continue;
// Only consider keypoints whose score is maximum in a local window. // Only consider keypoints whose score is maximum in a local window.
if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scores)) queue.enqueue({ score, part: { heatmapY, heatmapX, id: keypointId } }); if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scores)) queue.enqueue({ score, part: { heatmapY, heatmapX, id: keypointId } });
} }
@ -103,38 +104,37 @@ export function buildPartWithScoreQueue(scoreThreshold, scores) {
return queue; return queue;
} }
function withinRadius(poses, squaredNmsRadius, { x, y }, keypointId) { function withinRadius(poses, { x, y }, keypointId) {
return poses.some(({ keypoints }) => { return poses.some(({ keypoints }) => {
const correspondingKeypoint = keypoints[keypointId].position; const correspondingKeypoint = keypoints[keypointId].position;
return utils.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= squaredNmsRadius; return utils.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= squaredNmsRadius;
}); });
} }
function getInstanceScore(existingPoses, squaredNmsRadius, instanceKeypoints) { function getInstanceScore(existingPoses, instanceKeypoints) {
const notOverlappedKeypointScores = instanceKeypoints.reduce((result, { position, score }, keypointId) => { const notOverlappedKeypointScores = instanceKeypoints.reduce((result, { position, score }, keypointId) => {
if (!withinRadius(existingPoses, squaredNmsRadius, position, keypointId)) result += score; if (!withinRadius(existingPoses, position, keypointId)) result += score;
return result; return result;
}, 0.0); }, 0.0);
return notOverlappedKeypointScores / instanceKeypoints.length; return notOverlappedKeypointScores / instanceKeypoints.length;
} }
export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, nmsRadius, maxDetections, scoreThreshold) { export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, maxDetected, minConfidence) {
const poses: Array<{ keypoints: any, box: any, score: number }> = []; const poses: Array<{ keypoints: any, box: any, score: number }> = [];
const queue = buildPartWithScoreQueue(scoreThreshold, scoresBuffer); const queue = buildPartWithScoreQueue(minConfidence, scoresBuffer);
const squaredNmsRadius = nmsRadius ** 2; // Generate at most maxDetected object instances per image in decreasing root part score order.
// Generate at most maxDetections object instances per image in decreasing root part score order. while (poses.length < maxDetected && !queue.empty()) {
while (poses.length < maxDetections && !queue.empty()) {
// The top element in the queue is the next root candidate. // The top element in the queue is the next root candidate.
const root = queue.dequeue(); const root = queue.dequeue();
// Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance. // Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer); const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
if (withinRadius(poses, squaredNmsRadius, rootImageCoords, root.part.id)) continue; if (withinRadius(poses, rootImageCoords, root.part.id)) continue;
// Else start a new detection instance at the position of the root. // Else start a new detection instance at the position of the root.
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer); const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
const keypoints = allKeypoints.filter((a) => a.score > scoreThreshold); const keypoints = allKeypoints.filter((a) => a.score > minConfidence);
const score = getInstanceScore(poses, squaredNmsRadius, keypoints); const score = getInstanceScore(poses, keypoints);
const box = utils.getBoundingBox(keypoints); const box = utils.getBoundingBox(keypoints);
if (score > scoreThreshold) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 }); if (score > minConfidence) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });
} }
return poses; return poses;
} }

6
src/profile.ts
View File

@ -4,7 +4,7 @@ export const data = {};
export function run(modelName: string, profileData: any): void { export function run(modelName: string, profileData: any): void {
if (!profileData || !profileData.kernels) return; if (!profileData || !profileData.kernels) return;
const maxResults = 5; const maxDetected = 5;
const time = profileData.kernels const time = profileData.kernels
.filter((a) => a.kernelTimeMs > 0) .filter((a) => a.kernelTimeMs > 0)
.reduce((a, b) => a += b.kernelTimeMs, 0); .reduce((a, b) => a += b.kernelTimeMs, 0);
@ -16,8 +16,8 @@ export function run(modelName: string, profileData: any): void {
.map((a, i) => { a.id = i; return a; }) .map((a, i) => { a.id = i; return a; })
.filter((a) => a.totalBytesSnapshot > 0) .filter((a) => a.totalBytesSnapshot > 0)
.sort((a, b) => b.totalBytesSnapshot - a.totalBytesSnapshot); .sort((a, b) => b.totalBytesSnapshot - a.totalBytesSnapshot);
if (slowest.length > maxResults) slowest.length = maxResults; if (slowest.length > maxDetected) slowest.length = maxDetected;
if (largest.length > maxResults) largest.length = maxResults; if (largest.length > maxDetected) largest.length = maxDetected;
data[modelName] = { data[modelName] = {
model: modelName, model: modelName,
newBytes: profileData.newBytes, newBytes: profileData.newBytes,

62
src/prototype/prototype.ts
View File

@ -1,62 +0,0 @@
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
/*
Prototype implementation for model processing
Must implement
- load()
- predict()
Must account for:
- image processing, tfjs profiling
*/
let model;
let last = { };
let skipped = Number.MAX_SAFE_INTEGER;
export async function load(config) {
if (!model) {
model = await tf.loadGraphModel(join(config.modelBasePath, config.prototype.modelPath));
if (!model || !model.modelUrl) log('load model failed:', config.prototype.modelPath);
else if (config.debug) log('load model:', model.modelUrl);
}
return model;
}
export async function predict(image, config) {
if (!model) return null;
if ((skipped < config.prototype.skipFrames) && config.videoOptimized && Object.keys(last).length > 0) {
skipped++;
return last;
}
if (config.videoOptimized) skipped = 0;
else skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => {
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const enhance = tf.mul(resize, [255.0]);
tf.dispose(resize);
let resT;
if (!config.profile) {
if (config.prototype.enabled) resT = await model.predict(enhance);
} else {
const profileT = config.prototype.enabled ? await tf.profile(() => model.predict(enhance)) : {};
resT = profileT.result.clone();
profileT.result.dispose();
profile.run('prototype', profileT);
}
enhance.dispose();
let obj = {};
if (resT) {
const data = resT.dataSync();
obj = { data };
tf.dispose(resT);
}
last = obj;
resolve(obj);
});
}

2
typedoc/assets/js/search.js
View File

File diff suppressed because one or more lines are too long

26
typedoc/classes/human.html
View File

@ -118,7 +118,6 @@
<li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#enhance" class="tsd-kind-icon">enhance</a></li> <li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#enhance" class="tsd-kind-icon">enhance</a></li>
<li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#load" class="tsd-kind-icon">load</a></li> <li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#load" class="tsd-kind-icon">load</a></li>
<li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#match" class="tsd-kind-icon">match</a></li> <li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#match" class="tsd-kind-icon">match</a></li>
<li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#profiledata" class="tsd-kind-icon">profile<wbr>Data</a></li>
<li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#similarity" class="tsd-kind-icon">similarity</a></li> <li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#similarity" class="tsd-kind-icon">similarity</a></li>
<li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#warmup" class="tsd-kind-icon">warmup</a></li> <li class="tsd-kind-method tsd-parent-kind-class"><a href="human.html#warmup" class="tsd-kind-icon">warmup</a></li>
</ul> </ul>
@ -713,28 +712,6 @@
</li> </li>
</ul> </ul>
</section> </section>
<section class="tsd-panel tsd-member tsd-kind-method tsd-parent-kind-class">
<a name="profiledata" class="tsd-anchor"></a>
<h3>profile<wbr>Data</h3>
<ul class="tsd-signatures tsd-kind-method tsd-parent-kind-class">
<li class="tsd-signature tsd-kind-icon">profile<wbr>Data<span class="tsd-signature-symbol">(</span><span class="tsd-signature-symbol">)</span><span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{}</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">{ </span>largestKernelOps<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>newBytes<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>newTensors<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>numKernelOps<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>peakBytes<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>slowestKernelOps<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>timeKernelOps<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol"> }</span></li>
</ul>
<ul class="tsd-descriptions">
<li class="tsd-description">
<aside class="tsd-sources">
</aside>
<div class="tsd-comment tsd-typography">
<div class="lead">
<p>Internal: ProfileData method returns last known profiling information</p>
<ul>
<li>Requires human.config.profile set to true</li>
</ul>
</div>
</div>
<h4 class="tsd-returns-title">Returns <span class="tsd-signature-symbol">{}</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">{ </span>largestKernelOps<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>newBytes<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>newTensors<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>numKernelOps<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>peakBytes<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>slowestKernelOps<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol">; </span>timeKernelOps<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">any</span><span class="tsd-signature-symbol"> }</span></h4>
</li>
</ul>
</section>
<section class="tsd-panel tsd-member tsd-kind-method tsd-parent-kind-class"> <section class="tsd-panel tsd-member tsd-kind-method tsd-parent-kind-class">
<a name="similarity" class="tsd-anchor"></a> <a name="similarity" class="tsd-anchor"></a>
<h3>similarity</h3> <h3>similarity</h3>
@ -866,9 +843,6 @@
<li class=" tsd-kind-method tsd-parent-kind-class"> <li class=" tsd-kind-method tsd-parent-kind-class">
<a href="human.html#match" class="tsd-kind-icon">match</a> <a href="human.html#match" class="tsd-kind-icon">match</a>
</li> </li>
<li class=" tsd-kind-method tsd-parent-kind-class">
<a href="human.html#profiledata" class="tsd-kind-icon">profile<wbr>Data</a>
</li>
<li class=" tsd-kind-method tsd-parent-kind-class"> <li class=" tsd-kind-method tsd-parent-kind-class">
<a href="human.html#similarity" class="tsd-kind-icon">similarity</a> <a href="human.html#similarity" class="tsd-kind-icon">similarity</a>
</li> </li>

116
typedoc/interfaces/config.html
View File

@ -89,7 +89,6 @@
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#async" class="tsd-kind-icon">async</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#async" class="tsd-kind-icon">async</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#backend" class="tsd-kind-icon">backend</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#backend" class="tsd-kind-icon">backend</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#body" class="tsd-kind-icon">body</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#body" class="tsd-kind-icon">body</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#deallocate" class="tsd-kind-icon">deallocate</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#debug" class="tsd-kind-icon">debug</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#debug" class="tsd-kind-icon">debug</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#face" class="tsd-kind-icon">face</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#face" class="tsd-kind-icon">face</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#filter" class="tsd-kind-icon">filter</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#filter" class="tsd-kind-icon">filter</a></li>
@ -97,8 +96,6 @@
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#hand" class="tsd-kind-icon">hand</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#hand" class="tsd-kind-icon">hand</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#modelbasepath" class="tsd-kind-icon">model<wbr>Base<wbr>Path</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#modelbasepath" class="tsd-kind-icon">model<wbr>Base<wbr>Path</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#object" class="tsd-kind-icon">object</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#object" class="tsd-kind-icon">object</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#profile" class="tsd-kind-icon">profile</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#scoped" class="tsd-kind-icon">scoped</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#videooptimized" class="tsd-kind-icon">video<wbr>Optimized</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#videooptimized" class="tsd-kind-icon">video<wbr>Optimized</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#warmup" class="tsd-kind-icon">warmup</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#warmup" class="tsd-kind-icon">warmup</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#wasmpath" class="tsd-kind-icon">wasm<wbr>Path</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="config.html#wasmpath" class="tsd-kind-icon">wasm<wbr>Path</a></li>
@ -136,7 +133,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="body" class="tsd-anchor"></a> <a name="body" class="tsd-anchor"></a>
<h3>body</h3> <h3>body</h3>
<div class="tsd-signature tsd-kind-icon">body<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>maxDetections<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>nmsRadius<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>scoreThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span></div> <div class="tsd-signature tsd-kind-icon">body<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>maxDetected<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
<div class="tsd-comment tsd-typography"> <div class="tsd-comment tsd-typography">
@ -144,10 +141,9 @@
<p>Controlls and configures all body detection specific options</p> <p>Controlls and configures all body detection specific options</p>
<ul> <ul>
<li>enabled: true/false</li> <li>enabled: true/false</li>
<li>modelPath: paths for both hand detector model and hand skeleton model</li> <li>modelPath: body pose model, can be absolute path or relative to modelBasePath</li>
<li>maxDetections: maximum number of people detected in the input, should be set to the minimum number for performance</li> <li>minConfidence: threshold for discarding a prediction</li>
<li>scoreThreshold: threshold for deciding when to remove people based on score in non-maximum suppression</li> <li>maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance</li>
<li>nmsRadius: threshold for deciding whether body parts overlap too much in non-maximum suppression</li>
</ul> </ul>
</div> </div>
</div> </div>
@ -158,32 +154,17 @@
<h5>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span></h5> <h5>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>max<wbr>Detections<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>max<wbr>Detected<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li>
<li class="tsd-parameter">
<h5>min<wbr>Confidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>model<wbr>Path<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span></h5> <h5>model<wbr>Path<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span></h5>
</li> </li>
<li class="tsd-parameter">
<h5>nms<wbr>Radius<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li>
<li class="tsd-parameter">
<h5>score<wbr>Threshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li>
</ul> </ul>
</div> </div>
</section> </section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="deallocate" class="tsd-anchor"></a>
<h3>deallocate</h3>
<div class="tsd-signature tsd-kind-icon">deallocate<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">boolean</span></div>
<aside class="tsd-sources">
</aside>
<div class="tsd-comment tsd-typography">
<div class="lead">
<p>Internal: Use aggressive GPU memory deallocator when backend is set to <code>webgl</code> or <code>humangl</code></p>
</div>
</div>
</section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="debug" class="tsd-anchor"></a> <a name="debug" class="tsd-anchor"></a>
<h3>debug</h3> <h3>debug</h3>
@ -199,7 +180,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="face" class="tsd-anchor"></a> <a name="face" class="tsd-anchor"></a>
<h3>face</h3> <h3>face</h3>
<div class="tsd-signature tsd-kind-icon">face<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>description<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>detector<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>iouThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>maxFaces<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>return<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>scoreThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>skipInitial<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>mesh<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol"> }</span></div> <div class="tsd-signature tsd-kind-icon">face<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>description<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>detector<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>iouThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>maxDetected<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>return<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>skipInitial<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>mesh<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
<div class="tsd-comment tsd-typography"> <div class="tsd-comment tsd-typography">
@ -209,15 +190,14 @@
<li>face detection, face mesh detection, age, gender, emotion detection and face description <li>face detection, face mesh detection, age, gender, emotion detection and face description
Parameters:</li> Parameters:</li>
<li>enabled: true/false</li> <li>enabled: true/false</li>
<li>modelPath: path for individual face model</li> <li>modelPath: path for each of face models</li>
<li>minConfidence: threshold for discarding a prediction</li>
<li>iouThreshold: ammount of overlap between two detected objects before one object is removed</li>
<li>maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance</li>
<li>rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles</li> <li>rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles</li>
<li>maxFaces: maximum number of faces detected in the input, should be set to the minimum number for performance</li>
<li>skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true</li> <li>skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true</li>
<li>skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle</li> <li>skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle</li>
<li>minConfidence: threshold for discarding a prediction</li> <li>return: return extracted face as tensor for futher user processing</li>
<li>iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression</li>
<li>scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression</li>
<li>return extracted face as tensor for futher user processing</li>
</ul> </ul>
</div> </div>
</div> </div>
@ -242,13 +222,13 @@
</ul> </ul>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>detector<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>iouThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>maxFaces<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>return<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>scoreThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>skipInitial<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol"> }</span></h5> <h5>detector<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>iouThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>maxDetected<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>return<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>skipInitial<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol"> }</span></h5>
<ul class="tsd-parameters"> <ul class="tsd-parameters">
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>iou<wbr>Threshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>iou<wbr>Threshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>max<wbr>Faces<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>max<wbr>Detected<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>min<wbr>Confidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>min<wbr>Confidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
@ -262,9 +242,6 @@
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span></h5> <h5>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span></h5>
</li> </li>
<li class="tsd-parameter">
<h5>score<wbr>Threshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>skip<wbr>Frames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>skip<wbr>Frames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li> </li>
@ -510,7 +487,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="hand" class="tsd-anchor"></a> <a name="hand" class="tsd-anchor"></a>
<h3>hand</h3> <h3>hand</h3>
<div class="tsd-signature tsd-kind-icon">hand<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>detector<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>iouThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>landmarks<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>maxHands<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>scoreThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>skeleton<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>skipInitial<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol"> }</span></div> <div class="tsd-signature tsd-kind-icon">hand<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>detector<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>iouThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>landmarks<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>maxDetected<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>skeleton<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>skipInitial<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
<div class="tsd-comment tsd-typography"> <div class="tsd-comment tsd-typography">
@ -518,15 +495,14 @@
<p>Controlls and configures all hand detection specific options</p> <p>Controlls and configures all hand detection specific options</p>
<ul> <ul>
<li>enabled: true/false</li> <li>enabled: true/false</li>
<li>modelPath: paths for both hand detector model and hand skeleton model</li> <li>landmarks: detect hand landmarks or just hand boundary box</li>
<li>modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath</li>
<li>minConfidence: threshold for discarding a prediction</li>
<li>iouThreshold: ammount of overlap between two detected objects before one object is removed</li>
<li>maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance</li>
<li>rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted</li> <li>rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted</li>
<li>skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true</li> <li>skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true</li>
<li>skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle</li> <li>skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle</li>
<li>minConfidence: threshold for discarding a prediction</li>
<li>iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression</li>
<li>scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression</li>
<li>maxHands: maximum number of hands detected in the input, should be set to the minimum number for performance</li>
<li>landmarks: detect hand landmarks or just hand boundary box</li>
</ul> </ul>
</div> </div>
</div> </div>
@ -551,7 +527,7 @@
<h5>landmarks<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span></h5> <h5>landmarks<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>max<wbr>Hands<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>max<wbr>Detected<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>min<wbr>Confidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>min<wbr>Confidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
@ -559,9 +535,6 @@
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span></h5> <h5>rotation<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span></h5>
</li> </li>
<li class="tsd-parameter">
<h5>score<wbr>Threshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>skeleton<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span></h5> <h5>skeleton<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span></h5>
<ul class="tsd-parameters"> <ul class="tsd-parameters">
@ -589,7 +562,7 @@
<div class="lead"> <div class="lead">
<p>Base model path (typically starting with file://, http:// or https://) for all models</p> <p>Base model path (typically starting with file://, http:// or https://) for all models</p>
<ul> <ul>
<li>individual modelPath values are joined to this path</li> <li>individual modelPath values are relative to this path</li>
</ul> </ul>
</div> </div>
</div> </div>
@ -597,16 +570,18 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="object" class="tsd-anchor"></a> <a name="object" class="tsd-anchor"></a>
<h3>object</h3> <h3>object</h3>
<div class="tsd-signature tsd-kind-icon">object<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>iouThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>maxResults<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span></div> <div class="tsd-signature tsd-kind-icon">object<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>enabled<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">boolean</span><span class="tsd-signature-symbol">; </span>iouThreshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>maxDetected<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>minConfidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>modelPath<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>skipFrames<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
<div class="tsd-comment tsd-typography"> <div class="tsd-comment tsd-typography">
<div class="lead"> <div class="lead">
<p>Controlls and configures all object detection specific options</p> <p>Controlls and configures all object detection specific options</p>
<ul> <ul>
<li>enabled: true/false</li>
<li>modelPath: object detection model, can be absolute path or relative to modelBasePath</li>
<li>minConfidence: minimum score that detection must have to return as valid object</li> <li>minConfidence: minimum score that detection must have to return as valid object</li>
<li>iouThreshold: ammount of overlap between two detected objects before one object is removed</li> <li>iouThreshold: ammount of overlap between two detected objects before one object is removed</li>
<li>maxResults: maximum number of detections to return</li> <li>maxDetected: maximum number of detections to return</li>
<li>skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true</li> <li>skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true</li>
</ul> </ul>
</div> </div>
@ -621,7 +596,7 @@
<h5>iou<wbr>Threshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>iou<wbr>Threshold<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>max<wbr>Results<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>max<wbr>Detected<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>min<wbr>Confidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>min<wbr>Confidence<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
@ -635,30 +610,6 @@
</ul> </ul>
</div> </div>
</section> </section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="profile" class="tsd-anchor"></a>
<h3>profile</h3>
<div class="tsd-signature tsd-kind-icon">profile<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">boolean</span></div>
<aside class="tsd-sources">
</aside>
<div class="tsd-comment tsd-typography">
<div class="lead">
<p>Collect and print profiling data during inference operations</p>
</div>
</div>
</section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="scoped" class="tsd-anchor"></a>
<h3>scoped</h3>
<div class="tsd-signature tsd-kind-icon">scoped<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">boolean</span></div>
<aside class="tsd-sources">
</aside>
<div class="tsd-comment tsd-typography">
<div class="lead">
<p>Internal: Run all inference operations in an explicit local scope run to avoid memory leaks</p>
</div>
</div>
</section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="videooptimized" class="tsd-anchor"></a> <a name="videooptimized" class="tsd-anchor"></a>
<h3>video<wbr>Optimized</h3> <h3>video<wbr>Optimized</h3>
@ -737,9 +688,6 @@
<li class=" tsd-kind-property tsd-parent-kind-interface"> <li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="config.html#body" class="tsd-kind-icon">body</a> <a href="config.html#body" class="tsd-kind-icon">body</a>
</li> </li>
<li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="config.html#deallocate" class="tsd-kind-icon">deallocate</a>
</li>
<li class=" tsd-kind-property tsd-parent-kind-interface"> <li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="config.html#debug" class="tsd-kind-icon">debug</a> <a href="config.html#debug" class="tsd-kind-icon">debug</a>
</li> </li>
@ -761,12 +709,6 @@
<li class=" tsd-kind-property tsd-parent-kind-interface"> <li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="config.html#object" class="tsd-kind-icon">object</a> <a href="config.html#object" class="tsd-kind-icon">object</a>
</li> </li>
<li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="config.html#profile" class="tsd-kind-icon">profile</a>
</li>
<li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="config.html#scoped" class="tsd-kind-icon">scoped</a>
</li>
<li class=" tsd-kind-property tsd-parent-kind-interface"> <li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="config.html#videooptimized" class="tsd-kind-icon">video<wbr>Optimized</a> <a href="config.html#videooptimized" class="tsd-kind-icon">video<wbr>Optimized</a>
</li> </li>

54
types/config.d.ts vendored
View File

@ -12,12 +12,6 @@ export interface Config {
debug: boolean; debug: boolean;
/** Perform model loading and inference concurrently or sequentially */ /** Perform model loading and inference concurrently or sequentially */
async: boolean; async: boolean;
/** Collect and print profiling data during inference operations */
profile: boolean;
/** Internal: Use aggressive GPU memory deallocator when backend is set to `webgl` or `humangl` */
deallocate: boolean;
/** Internal: Run all inference operations in an explicit local scope run to avoid memory leaks */
scoped: boolean;
/** Perform additional optimizations when input is video, /** Perform additional optimizations when input is video,
* - must be disabled for images * - must be disabled for images
* - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs * - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs
@ -30,7 +24,7 @@ export interface Config {
*/ */
warmup: 'none' | 'face' | 'full' | 'body'; warmup: 'none' | 'face' | 'full' | 'body';
/** Base model path (typically starting with file://, http:// or https://) for all models /** Base model path (typically starting with file://, http:// or https://) for all models
* - individual modelPath values are joined to this path * - individual modelPath values are relative to this path
*/ */
modelBasePath: string; modelBasePath: string;
/** Run input through image filters before inference /** Run input through image filters before inference
@ -89,27 +83,25 @@ export interface Config {
* - face detection, face mesh detection, age, gender, emotion detection and face description * - face detection, face mesh detection, age, gender, emotion detection and face description
* Parameters: * Parameters:
* - enabled: true/false * - enabled: true/false
* - modelPath: path for individual face model * - modelPath: path for each of face models
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance
* - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles * - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles
* - maxFaces: maximum number of faces detected in the input, should be set to the minimum number for performance
* - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true * - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true
* - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle * - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle
* - minConfidence: threshold for discarding a prediction * - return: return extracted face as tensor for futher user processing
* - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
* - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
* - return extracted face as tensor for futher user processing
*/ */
face: { face: {
enabled: boolean; enabled: boolean;
detector: { detector: {
modelPath: string; modelPath: string;
rotation: boolean; rotation: boolean;
maxFaces: number; maxDetected: number;
skipFrames: number; skipFrames: number;
skipInitial: boolean; skipInitial: boolean;
minConfidence: number; minConfidence: number;
iouThreshold: number; iouThreshold: number;
scoreThreshold: number;
return: boolean; return: boolean;
}; };
mesh: { mesh: {
@ -135,29 +127,26 @@ export interface Config {
}; };
/** Controlls and configures all body detection specific options /** Controlls and configures all body detection specific options
* - enabled: true/false * - enabled: true/false
* - modelPath: paths for both hand detector model and hand skeleton model * - modelPath: body pose model, can be absolute path or relative to modelBasePath
* - maxDetections: maximum number of people detected in the input, should be set to the minimum number for performance * - minConfidence: threshold for discarding a prediction
* - scoreThreshold: threshold for deciding when to remove people based on score in non-maximum suppression * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance
* - nmsRadius: threshold for deciding whether body parts overlap too much in non-maximum suppression
*/ */
body: { body: {
enabled: boolean; enabled: boolean;
modelPath: string; modelPath: string;
maxDetections: number; maxDetected: number;
scoreThreshold: number; minConfidence: number;
nmsRadius: number;
}; };
/** Controlls and configures all hand detection specific options /** Controlls and configures all hand detection specific options
* - enabled: true/false * - enabled: true/false
* - modelPath: paths for both hand detector model and hand skeleton model * - landmarks: detect hand landmarks or just hand boundary box
* - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance
* - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
* - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true * - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true
* - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle * - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
* - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
* - maxHands: maximum number of hands detected in the input, should be set to the minimum number for performance
* - landmarks: detect hand landmarks or just hand boundary box
*/ */
hand: { hand: {
enabled: boolean; enabled: boolean;
@ -166,8 +155,7 @@ export interface Config {
skipInitial: boolean; skipInitial: boolean;
minConfidence: number; minConfidence: number;
iouThreshold: number; iouThreshold: number;
scoreThreshold: number; maxDetected: number;
maxHands: number;
landmarks: boolean; landmarks: boolean;
detector: { detector: {
modelPath: string; modelPath: string;
@ -177,9 +165,11 @@ export interface Config {
}; };
}; };
/** Controlls and configures all object detection specific options /** Controlls and configures all object detection specific options
* - enabled: true/false
* - modelPath: object detection model, can be absolute path or relative to modelBasePath
* - minConfidence: minimum score that detection must have to return as valid object * - minConfidence: minimum score that detection must have to return as valid object
* - iouThreshold: ammount of overlap between two detected objects before one object is removed * - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxResults: maximum number of detections to return * - maxDetected: maximum number of detections to return
* - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true * - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true
*/ */
object: { object: {
@ -187,7 +177,7 @@ export interface Config {
modelPath: string; modelPath: string;
minConfidence: number; minConfidence: number;
iouThreshold: number; iouThreshold: number;
maxResults: number; maxDetected: number;
skipFrames: number; skipFrames: number;
}; };
} }

12
types/human.d.ts vendored
View File

@ -111,18 +111,6 @@ export declare class Human {
* - @param userConfig: {@link Config} * - @param userConfig: {@link Config}
*/ */
constructor(userConfig?: Config | Object); constructor(userConfig?: Config | Object);
/** Internal: ProfileData method returns last known profiling information
* - Requires human.config.profile set to true
*/
profileData(): {
newBytes: any;
newTensors: any;
peakBytes: any;
numKernelOps: any;
timeKernelOps: any;
slowestKernelOps: any;
largestKernelOps: any;
} | {};
/** @hidden */ /** @hidden */
analyze: (...msg: any[]) => void; analyze: (...msg: any[]) => void;
/** Simmilarity method calculates simmilarity between two provided face descriptors (face embeddings) /** Simmilarity method calculates simmilarity between two provided face descriptors (face embeddings)

4
types/posenet/poses.d.ts vendored
View File

@ -1,7 +1,7 @@
import * as utils from './utils'; import * as utils from './utils';
export declare function decodePose(root: any, scores: any, offsets: any, outputStride: any, displacementsFwd: any, displacementsBwd: any): any[]; export declare function decodePose(root: any, scores: any, offsets: any, outputStride: any, displacementsFwd: any, displacementsBwd: any): any[];
export declare function buildPartWithScoreQueue(scoreThreshold: any, scores: any): utils.MaxHeap; export declare function buildPartWithScoreQueue(minConfidence: any, scores: any): utils.MaxHeap;
export declare function decode(offsetsBuffer: any, scoresBuffer: any, displacementsFwdBuffer: any, displacementsBwdBuffer: any, nmsRadius: any, maxDetections: any, scoreThreshold: any): { export declare function decode(offsetsBuffer: any, scoresBuffer: any, displacementsFwdBuffer: any, displacementsBwdBuffer: any, maxDetected: any, minConfidence: any): {
keypoints: any; keypoints: any;
box: any; box: any;
score: number; score: number;

2
types/profile.d.ts vendored
View File

@ -1,2 +0,0 @@
export declare const data: {};
export declare function run(modelName: string, profileData: any): void;

2
wiki

@ -1 +1 @@
Subproject commit 3b81af15f2560de5c06f20cbd8de57caf62682f2 Subproject commit 906244487754b61fd24f49fe2db91ea68264137d