ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

major update for 1.8 release candidate

pull/280/head
Vladimir Mandic 2021-04-25 13:16:04 -04:00
parent 439a7af2e4
commit 3c11ef4189
18 changed files with 140 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@ -1,6 +1,6 @@
# @vladmandic/human
Version: **1.7.1**
Version: **1.8.0**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>**
@ -9,11 +9,12 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog
### **HEAD -> main** 2021/04/25 mandic00@live.com
### **1.7.1** 2021/04/25 mandic00@live.com
### **origin/main** 2021/04/24 mandic00@live.com
- remove obsolete binary models
- enable cross origin isolation
- rewrite posenet decoder
- remove efficientpose

19
TODO.md
View File

@ -17,4 +17,21 @@ N/A
- Blazepose
Needs detector before running pose to center the image
## Soon to be Removed
## RC: 1.8
### Done
Major configuration simplification:
- Unified minConfidence and scoreThresdold as minConfidence
- Replaced nmsRadius with built-in default
- Replaced maxFaces, maxDetections, maxHands, maxResults with maxDetected
- Remove deallocate, profile, scoped
Stop building sourcemaps for NodeJS deliverables
### TBD
- Remove modelPaths
- Remove blazeface-front, replace blazeface-back with blazeface
- NodeJS Exception handling

15
demo/index.js
View File

@ -492,15 +492,13 @@ function setupMenu() {
menu.process = new Menu(document.body, '', { top, left: x[2] });
menu.process.addList('backend', ['cpu', 'webgl', 'wasm', 'humangl'], human.config.backend, (val) => human.config.backend = val);
menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val);
// menu.process.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val);
// menu.process.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val);
menu.process.addBool('use web worker', ui, 'useWorker');
menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.process.addLabel('model parameters');
menu.process.addRange('max objects', human.config.face.detector, 'maxFaces', 1, 50, 1, (val) => {
human.config.face.detector.maxFaces = parseInt(val);
human.config.body.maxDetections = parseInt(val);
human.config.hand.maxHands = parseInt(val);
menu.process.addRange('max objects', human.config.face.detector, 'maxDetected', 1, 50, 1, (val) => {
human.config.face.detector.maxDetected = parseInt(val);
human.config.body.maxDetected = parseInt(val);
human.config.hand.maxDetected = parseInt(val);
});
menu.process.addRange('skip frames', human.config.face.detector, 'skipFrames', 0, 50, 1, (val) => {
human.config.face.detector.skipFrames = parseInt(val);
@ -512,11 +510,6 @@ function setupMenu() {
human.config.face.emotion.minConfidence = parseFloat(val);
human.config.hand.minConfidence = parseFloat(val);
});
menu.process.addRange('score threshold', human.config.face.detector, 'scoreThreshold', 0.1, 1.0, 0.05, (val) => {
human.config.face.detector.scoreThreshold = parseFloat(val);
human.config.hand.scoreThreshold = parseFloat(val);
human.config.body.scoreThreshold = parseFloat(val);
});
menu.process.addRange('overlap', human.config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => {
human.config.face.detector.iouThreshold = parseFloat(val);
human.config.hand.iouThreshold = parseFloat(val);

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "@vladmandic/human",
"version": "1.7.1",
"version": "1.8.0",
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
"sideEffects": false,
"main": "dist/human.node.js",

10
src/age/age.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
let model;
let last = { age: 0 };
@ -31,14 +30,7 @@ export async function predict(image, config) {
let ageT;
const obj = { age: 0 };
if (!config.profile) {
if (config.face.age.enabled) ageT = await model.predict(enhance);
} else {
const profileAge = config.face.age.enabled ? await tf.profile(() => model.predict(enhance)) : {};
ageT = profileAge.result.clone();
profileAge.result.dispose();
profile.run('age', profileAge);
}
if (config.face.age.enabled) ageT = await model.predict(enhance);
enhance.dispose();
if (ageT) {

148
src/config.ts
View File

@ -9,33 +9,34 @@
export interface Config {
/** Backend used for TFJS operations */
backend: null | '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow',
/** Path to *.wasm files if backend is set to `wasm` */
wasmPath: string,
/** Print debug statements to console */
debug: boolean,
/** Perform model loading and inference concurrently or sequentially */
async: boolean,
/** Collect and print profiling data during inference operations */
profile: boolean,
/** Internal: Use aggressive GPU memory deallocator when backend is set to `webgl` or `humangl` */
deallocate: boolean,
/** Internal: Run all inference operations in an explicit local scope run to avoid memory leaks */
scoped: boolean,
/** Perform additional optimizations when input is video,
* - must be disabled for images
* - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs
* - skips boundary detection for every `skipFrames` frames specified for each model
* - while maintaining in-box detection since objects don't change definition as fast */
videoOptimized: boolean,
/** What to use for `human.warmup()`
* - warmup pre-initializes all models for faster inference but can take significant time on startup
* - only used for `webgl` and `humangl` backends
*/
warmup: 'none' | 'face' | 'full' | 'body',
/** Base model path (typically starting with file://, http:// or https://) for all models
* - individual modelPath values are joined to this path
* - individual modelPath values are relative to this path
*/
modelBasePath: string,
/** Run input through image filters before inference
* - image filters run with near-zero latency as they are executed on the GPU
*/
@ -90,31 +91,30 @@ export interface Config {
gesture: {
enabled: boolean,
},
/** Controlls and configures all face-specific options:
* - face detection, face mesh detection, age, gender, emotion detection and face description
* Parameters:
* - enabled: true/false
* - modelPath: path for individual face model
* - modelPath: path for each of face models
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance
* - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles
* - maxFaces: maximum number of faces detected in the input, should be set to the minimum number for performance
* - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true
* - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
* - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
* - return extracted face as tensor for futher user processing
* - return: return extracted face as tensor for futher user processing
*/
face: {
enabled: boolean,
detector: {
modelPath: string,
rotation: boolean,
maxFaces: number,
maxDetected: number,
skipFrames: number,
skipInitial: boolean,
minConfidence: number,
iouThreshold: number,
scoreThreshold: number,
return: boolean,
},
mesh: {
@ -138,31 +138,30 @@ export interface Config {
modelPath: string,
},
},
/** Controlls and configures all body detection specific options
* - enabled: true/false
* - modelPath: paths for both hand detector model and hand skeleton model
* - maxDetections: maximum number of people detected in the input, should be set to the minimum number for performance
* - scoreThreshold: threshold for deciding when to remove people based on score in non-maximum suppression
* - nmsRadius: threshold for deciding whether body parts overlap too much in non-maximum suppression
* - modelPath: body pose model, can be absolute path or relative to modelBasePath
* - minConfidence: threshold for discarding a prediction
* - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance
*/
body: {
enabled: boolean,
modelPath: string,
maxDetections: number,
scoreThreshold: number,
nmsRadius: number,
maxDetected: number,
minConfidence: number,
},
/** Controlls and configures all hand detection specific options
* - enabled: true/false
* - modelPath: paths for both hand detector model and hand skeleton model
* - landmarks: detect hand landmarks or just hand boundary box
* - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance
* - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
* - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true
* - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle
* - minConfidence: threshold for discarding a prediction
* - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
* - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
* - maxHands: maximum number of hands detected in the input, should be set to the minimum number for performance
* - landmarks: detect hand landmarks or just hand boundary box
*/
hand: {
enabled: boolean,
@ -171,8 +170,7 @@ export interface Config {
skipInitial: boolean,
minConfidence: number,
iouThreshold: number,
scoreThreshold: number,
maxHands: number,
maxDetected: number,
landmarks: boolean,
detector: {
modelPath: string,
@ -181,10 +179,13 @@ export interface Config {
modelPath: string,
},
},
/** Controlls and configures all object detection specific options
* - enabled: true/false
* - modelPath: object detection model, can be absolute path or relative to modelBasePath
* - minConfidence: minimum score that detection must have to return as valid object
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxResults: maximum number of detections to return
* - maxDetected: maximum number of detections to return
* - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true
*/
object: {
@ -192,40 +193,20 @@ export interface Config {
modelPath: string,
minConfidence: number,
iouThreshold: number,
maxResults: number,
maxDetected: number,
skipFrames: number,
},
}
const config: Config = {
backend: 'webgl', // select tfjs backend to use
backend: 'webgl', // select tfjs backend to use, leave empty to use default backend
// can be 'webgl', 'wasm', 'cpu', or 'humangl' which is a custom version of webgl
// leave as empty string to continue using default backend
// when backend is set outside of Human library
modelBasePath: '../models/', // base path for all models
wasmPath: '../assets/', // path for wasm binaries
// only used for backend: wasm
wasmPath: '../assets/', // path for wasm binariesm, only used for backend: wasm
debug: true, // print additional status messages to console
async: true, // execute enabled models in parallel
// this disables per-model performance data but
// slightly increases performance
// cannot be used if profiling is enabled
profile: false, // internal: enable tfjs profiling
// this has significant performance impact
// only enable for debugging purposes
// currently only implemented for age,gender,emotion models
deallocate: false, // internal: aggresively deallocate gpu memory after each usage
// only valid for webgl and humangl backend and only during first call
// cannot be changed unless library is reloaded
// this has significant performance impact
// only enable on low-memory devices
scoped: false, // internal: enable scoped runs
// some models *may* have memory leaks,
// this wrapps everything in a local scope at a cost of performance
// typically not needed
videoOptimized: true, // perform additional optimizations when input is video,
// must be disabled for images
// automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs
// automatically disabled for Image, ImageData, ImageBitmap
// skips boundary detection for every n frames
// while maintaining in-box detection since objects cannot move that fast
warmup: 'face', // what to use for human.warmup(), can be 'none', 'face', 'full'
@ -258,7 +239,7 @@ const config: Config = {
},
gesture: {
enabled: true, // enable simple gesture recognition
enabled: true, // enable gesture recognition based on model results
},
face: {
@ -267,12 +248,11 @@ const config: Config = {
// detector, mesh, iris, age, gender, emotion
// (note: module is not loaded until it is required)
detector: {
modelPath: 'blazeface-back.json', // detector model
// can be either absolute path or relative to modelBasePath
modelPath: 'blazeface-back.json', // detector model, can be absolute path or relative to modelBasePath
rotation: false, // use best-guess rotated face image or just box with rotation as-is
// false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees
// this parameter is not valid in nodejs
maxFaces: 10, // maximum number of faces detected in the input
maxDetected: 10, // maximum number of faces detected in the input
// should be set to the minimum number for performance
skipFrames: 21, // how many frames to go without re-running the face bounding box detector
// only used for video inputs
@ -282,18 +262,13 @@ const config: Config = {
skipInitial: false, // if previous detection resulted in no faces detected,
// should skipFrames be reset immediately to force new detection cycle
minConfidence: 0.2, // threshold for discarding a prediction
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in
// non-maximum suppression (0.1 means drop if overlap 10%)
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score
// in non-maximum suppression,
// this is applied on detection objects only and before minConfidence
iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed
return: false, // return extracted face as tensor
},
mesh: {
enabled: true,
modelPath: 'facemesh.json', // facemesh model
// can be either absolute path or relative to modelBasePath
modelPath: 'facemesh.json', // facemesh model, can be absolute path or relative to modelBasePath
},
iris: {
@ -316,25 +291,18 @@ const config: Config = {
enabled: true,
minConfidence: 0.1, // threshold for discarding a prediction
skipFrames: 32, // how many frames to go without re-running the detector
modelPath: 'emotion.json', // face emotion model
// can be either absolute path or relative to modelBasePath
modelPath: 'emotion.json', // face emotion model, can be absolute path or relative to modelBasePath
},
},
body: {
enabled: true,
modelPath: 'posenet.json', // body model
// can be either absolute path or relative to modelBasePath
// can be 'posenet', 'blazepose' or 'efficientpose'
// 'blazepose' and 'efficientpose' are experimental
maxDetections: 1, // maximum number of people detected in the input
modelPath: 'posenet.json', // body model, can be absolute path or relative to modelBasePath
// can be 'posenet' or 'blazepose'
maxDetected: 1, // maximum number of people detected in the input
// should be set to the minimum number for performance
// only valid for posenet as blazepose only detects single pose
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score
// in non-maximum suppression
// only valid for posenet as blazepose only detects single pose
nmsRadius: 20, // radius for deciding points are too close in non-maximum suppression
// only valid for posenet as blazepose only detects single pose
minConfidence: 0.2, // threshold for discarding a prediction
},
hand: {
@ -349,32 +317,24 @@ const config: Config = {
skipInitial: false, // if previous detection resulted in no hands detected,
// should skipFrames be reset immediately to force new detection cycle
minConfidence: 0.1, // threshold for discarding a prediction
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much
// in non-maximum suppression
scoreThreshold: 0.5, // threshold for deciding when to remove boxes based on
// score in non-maximum suppression
maxHands: 1, // maximum number of hands detected in the input
iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed
maxDetected: 1, // maximum number of hands detected in the input
// should be set to the minimum number for performance
landmarks: true, // detect hand landmarks or just hand boundary box
detector: {
modelPath: 'handdetect.json', // hand detector model
// can be either absolute path or relative to modelBasePath
modelPath: 'handdetect.json', // hand detector model, can be absolute path or relative to modelBasePath
},
skeleton: {
modelPath: 'handskeleton.json', // hand skeleton model
// can be either absolute path or relative to modelBasePath
modelPath: 'handskeleton.json', // hand skeleton model, can be absolute path or relative to modelBasePath
},
},
object: {
enabled: false,
modelPath: 'nanodet.json', // object detection model
// can be either absolute path or relative to modelBasePath
// 'nanodet' is experimental
minConfidence: 0.20, // threshold for discarding a prediction
iouThreshold: 0.40, // threshold for deciding whether boxes overlap too much
// in non-maximum suppression
maxResults: 10, // maximum number of objects detected in the input
modelPath: 'nanodet.json', // experimental: object detection model, can be absolute path or relative to modelBasePath
minConfidence: 0.2, // threshold for discarding a prediction
iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed
maxDetected: 10, // maximum number of objects detected in the input
skipFrames: 41, // how many frames to go without re-running the detector
},
};

48
src/draw/draw.ts
View File

@ -294,68 +294,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, draw
// shoulder line
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// torso main
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
// leg left
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// leg right
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// arm left
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// arm right
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// draw all
}

15
src/efficientpose/efficientpose.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
let model;
let keypoints: Array<any> = [];
@ -55,15 +54,7 @@ export async function predict(image, config) {
});
let resT;
if (!config.profile) {
if (config.body.enabled) resT = await model.predict(tensor);
} else {
const profileT = config.body.enabled ? await tf.profile(() => model.predict(tensor)) : {};
resT = profileT.result.clone();
profileT.result.dispose();
profile.run('body', profileT);
}
if (config.body.enabled) resT = await model.predict(tensor);
tensor.dispose();
if (resT) {
@ -76,8 +67,8 @@ export async function predict(image, config) {
// process each unstacked tensor as a separate body part
for (let id = 0; id < stack.length; id++) {
// actual processing to get coordinates and score
const [x, y, score] = max2d(stack[id], config.body.scoreThreshold);
if (score > config.body.scoreThreshold) {
const [x, y, score] = max2d(stack[id], config.body.minConfidence);
if (score > config.body.minConfidence) {
parts.push({
id,
score: Math.round(100 * score) / 100,

15
src/emotion/emotion.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'];
let model;
@ -46,17 +45,9 @@ export async function predict(image, config) {
grayscale.dispose();
const obj: Array<{ score: number, emotion: string }> = [];
if (config.face.emotion.enabled) {
let data;
if (!config.profile) {
const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation
data = emotionT.dataSync();
tf.dispose(emotionT);
} else {
const profileData = await tf.profile(() => model.predict(normalize));
data = profileData.result.dataSync();
profileData.result.dispose();
profile.run('emotion', profileData);
}
const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation
const data = emotionT.dataSync();
tf.dispose(emotionT);
for (let i = 0; i < data.length; i++) {
if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
}

9
src/faceres/faceres.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
let model;
let last = { age: 0 };
@ -108,13 +107,7 @@ export async function predict(image, config) {
genderConfidence: <number>0,
descriptor: <number[]>[] };
if (!config.profile) {
if (config.face.description.enabled) resT = await model.predict(enhanced);
} else {
const profileDesc = config.face.description.enabled ? await tf.profile(() => model.predict(enhanced)) : {};
resT = profileDesc.result;
profile.run('faceres', profileDesc);
}
if (config.face.description.enabled) resT = await model.predict(enhanced);
tf.dispose(enhanced);
if (resT) {

10
src/gender/gender.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
let model;
let last = { gender: '' };
@ -49,14 +48,7 @@ export async function predict(image, config) {
let genderT;
const obj = { gender: '', confidence: 0 };
if (!config.profile) {
if (config.face.gender.enabled) genderT = await model.predict(enhance);
} else {
const profileGender = config.face.gender.enabled ? await tf.profile(() => model.predict(enhance)) : {};
genderT = profileGender.result.clone();
profileGender.result.dispose();
profile.run('gender', profileGender);
}
if (config.face.gender.enabled) genderT = await model.predict(enhance);
enhance.dispose();
if (genderT) {

2
src/handpose/handdetector.ts
View File

@ -46,7 +46,7 @@ export class HandDetector {
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
rawBoxes.dispose();
const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxHands, config.hand.iouThreshold, config.hand.scoreThreshold);
const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
const filtered = filteredT.arraySync();
scoresT.dispose();

2
src/handpose/handpipeline.ts
View File

@ -83,7 +83,7 @@ export class HandPipeline {
if (config.videoOptimized) this.skipped++;
// if detector result count doesn't match current working set, use it to reset current working set
if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxHands) || !config.hand.landmarks)) {
if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxDetected) || !config.hand.landmarks)) {
this.detectedHands = 0;
this.storedBoxes = [...boxes];
// for (const possible of boxes) this.storedBoxes.push(possible);

21
src/human.ts
View File

@ -13,7 +13,6 @@ import * as nanodet from './nanodet/nanodet';
import * as gesture from './gesture/gesture';
import * as image from './image/image';
import * as draw from './draw/draw';
import * as profile from './profile';
import { Config, defaults } from './config';
import { Result } from './result';
import * as sample from './sample';
@ -168,14 +167,6 @@ export class Human {
this.sysinfo = sysinfo.info();
}
/** Internal: ProfileData method returns last known profiling information
* - Requires human.config.profile set to true
*/
profileData(): { newBytes, newTensors, peakBytes, numKernelOps, timeKernelOps, slowestKernelOps, largestKernelOps } | {} {
if (this.config.profile) return profile.data;
return {};
}
// helper function: measure tensor leak
/** @hidden */
analyze = (...msg) => {
@ -335,9 +326,9 @@ export class Human {
if (this.tf.getBackend() === 'webgl' || this.tf.getBackend() === 'humangl') {
this.tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false);
this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
if (this.config.deallocate) {
log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate);
this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1);
if (typeof this.config['deallocate'] !== 'undefined') {
log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true);
this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', 0);
}
const gl = await this.tf.backend().getGPGPUContext().gl;
if (this.config.debug) log(`gl version:${gl.getParameter(gl.VERSION)} renderer:${gl.getParameter(gl.RENDERER)}`);
@ -378,9 +369,6 @@ export class Human {
// load models if enabled
await this.load();
if (this.config.scoped) this.tf.engine().startScope();
this.analyze('Start Scope:');
// disable video optimization for inputs of type image, but skip if inside worker thread
let previousVideoOptimized;
// @ts-ignore ignore missing type for WorkerGlobalScope as that is the point
@ -474,9 +462,6 @@ export class Human {
}
tf.dispose(process.tensor);
if (this.config.scoped) this.tf.engine().endScope();
this.analyze('End Scope:');
// run gesture analysis last
let gestureRes: any[] = [];
if (this.config.gesture.enabled) {

11
src/nanodet/nanodet.ts
View File

@ -1,6 +1,5 @@
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
import { labels } from './labels';
let model;
@ -83,7 +82,7 @@ async function process(res, inputSize, outputShape, config) {
const nmsScores = results.map((a) => a.score);
let nmsIdx: any[] = [];
if (nmsBoxes && nmsBoxes.length > 0) {
const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxResults, config.object.iouThreshold, config.object.minConfidence);
const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
nmsIdx = nms.dataSync();
tf.dispose(nms);
}
@ -114,13 +113,7 @@ export async function predict(image, config) {
resize.dispose();
let objectT;
if (!config.profile) {
if (config.object.enabled) objectT = await model.predict(transpose);
} else {
const profileObject = config.object.enabled ? await tf.profile(() => model.predict(transpose)) : {};
objectT = profileObject.result;
profile.run('object', profileObject);
}
if (config.object.enabled) objectT = await model.predict(transpose);
transpose.dispose();
const obj = await process(objectT, model.inputSize, outputSize, config);

28
src/posenet/poses.ts
View File

@ -3,6 +3,7 @@ import * as kpt from './keypoints';
const localMaximumRadius = 1;
const defaultOutputStride = 16;
const squaredNmsRadius = 20 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
const getDisplacement = (point) => ({
@ -86,7 +87,7 @@ function scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scor
return localMaximum;
}
export function buildPartWithScoreQueue(scoreThreshold, scores) {
export function buildPartWithScoreQueue(minConfidence, scores) {
const [height, width, numKeypoints] = scores.shape;
const queue = new utils.MaxHeap(height * width * numKeypoints, ({ score }) => score);
for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
@ -94,7 +95,7 @@ export function buildPartWithScoreQueue(scoreThreshold, scores) {
for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
const score = scores.get(heatmapY, heatmapX, keypointId);
// Only consider parts with score greater or equal to threshold as root candidates.
if (score < scoreThreshold) continue;
if (score < minConfidence) continue;
// Only consider keypoints whose score is maximum in a local window.
if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scores)) queue.enqueue({ score, part: { heatmapY, heatmapX, id: keypointId } });
}
@ -103,38 +104,37 @@ export function buildPartWithScoreQueue(scoreThreshold, scores) {
return queue;
}
function withinRadius(poses, squaredNmsRadius, { x, y }, keypointId) {
function withinRadius(poses, { x, y }, keypointId) {
return poses.some(({ keypoints }) => {
const correspondingKeypoint = keypoints[keypointId].position;
return utils.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= squaredNmsRadius;
});
}
function getInstanceScore(existingPoses, squaredNmsRadius, instanceKeypoints) {
function getInstanceScore(existingPoses, instanceKeypoints) {
const notOverlappedKeypointScores = instanceKeypoints.reduce((result, { position, score }, keypointId) => {
if (!withinRadius(existingPoses, squaredNmsRadius, position, keypointId)) result += score;
if (!withinRadius(existingPoses, position, keypointId)) result += score;
return result;
}, 0.0);
return notOverlappedKeypointScores / instanceKeypoints.length;
}
export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, nmsRadius, maxDetections, scoreThreshold) {
export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, maxDetected, minConfidence) {
const poses: Array<{ keypoints: any, box: any, score: number }> = [];
const queue = buildPartWithScoreQueue(scoreThreshold, scoresBuffer);
const squaredNmsRadius = nmsRadius ** 2;
// Generate at most maxDetections object instances per image in decreasing root part score order.
while (poses.length < maxDetections && !queue.empty()) {
const queue = buildPartWithScoreQueue(minConfidence, scoresBuffer);
// Generate at most maxDetected object instances per image in decreasing root part score order.
while (poses.length < maxDetected && !queue.empty()) {
// The top element in the queue is the next root candidate.
const root = queue.dequeue();
// Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
if (withinRadius(poses, squaredNmsRadius, rootImageCoords, root.part.id)) continue;
if (withinRadius(poses, rootImageCoords, root.part.id)) continue;
// Else start a new detection instance at the position of the root.
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
const keypoints = allKeypoints.filter((a) => a.score > scoreThreshold);
const score = getInstanceScore(poses, squaredNmsRadius, keypoints);
const keypoints = allKeypoints.filter((a) => a.score > minConfidence);
const score = getInstanceScore(poses, keypoints);
const box = utils.getBoundingBox(keypoints);
if (score > scoreThreshold) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });
if (score > minConfidence) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });
}
return poses;
}

6
src/profile.ts
View File

@ -4,7 +4,7 @@ export const data = {};
export function run(modelName: string, profileData: any): void {
if (!profileData || !profileData.kernels) return;
const maxResults = 5;
const maxDetected = 5;
const time = profileData.kernels
.filter((a) => a.kernelTimeMs > 0)
.reduce((a, b) => a += b.kernelTimeMs, 0);
@ -16,8 +16,8 @@ export function run(modelName: string, profileData: any): void {
.map((a, i) => { a.id = i; return a; })
.filter((a) => a.totalBytesSnapshot > 0)
.sort((a, b) => b.totalBytesSnapshot - a.totalBytesSnapshot);
if (slowest.length > maxResults) slowest.length = maxResults;
if (largest.length > maxResults) largest.length = maxResults;
if (slowest.length > maxDetected) slowest.length = maxDetected;
if (largest.length > maxDetected) largest.length = maxDetected;
data[modelName] = {
model: modelName,
newBytes: profileData.newBytes,

2
wiki

@ -1 +1 @@
Subproject commit 3b81af15f2560de5c06f20cbd8de57caf62682f2
Subproject commit 906244487754b61fd24f49fe2db91ea68264137d