From 3c11ef41897d6987d054c3af3f715ed94cea207c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 25 Apr 2021 13:16:04 -0400 Subject: [PATCH] major update for 1.8 release candidate --- CHANGELOG.md | 9 +- TODO.md | 19 +++- demo/index.js | 15 +-- package.json | 2 +- src/age/age.ts | 10 +- src/config.ts | 148 +++++++++++------------------ src/draw/draw.ts | 48 +++++----- src/efficientpose/efficientpose.ts | 15 +-- src/emotion/emotion.ts | 15 +-- src/faceres/faceres.ts | 9 +- src/gender/gender.ts | 10 +- src/handpose/handdetector.ts | 2 +- src/handpose/handpipeline.ts | 2 +- src/human.ts | 21 +--- src/nanodet/nanodet.ts | 11 +-- src/posenet/poses.ts | 28 +++--- src/profile.ts | 6 +- wiki | 2 +- 18 files changed, 140 insertions(+), 232 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 096e14f9..732b06af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # @vladmandic/human -Version: **1.7.1** +Version: **1.8.0** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Author: **Vladimir Mandic ** @@ -9,11 +9,12 @@ Repository: **** ## Changelog +### **HEAD -> main** 2021/04/25 mandic00@live.com + + ### **1.7.1** 2021/04/25 mandic00@live.com - -### **origin/main** 2021/04/24 mandic00@live.com - +- remove obsolete binary models - enable cross origin isolation - rewrite posenet decoder - remove efficientpose diff --git a/TODO.md b/TODO.md index 5bd6eebd..49e38543 100644 --- a/TODO.md +++ b/TODO.md @@ -17,4 +17,21 @@ N/A - Blazepose Needs detector before running pose to center the image -## Soon to be Removed +## RC: 1.8 + +### Done + +Major configuration simplification: + +- Unified minConfidence and scoreThresdold as minConfidence +- Replaced nmsRadius with built-in default +- Replaced maxFaces, maxDetections, maxHands, maxResults with maxDetected +- Remove deallocate, profile, scoped + +Stop building sourcemaps for NodeJS deliverables + +### TBD + +- Remove modelPaths +- Remove blazeface-front, replace blazeface-back with blazeface +- NodeJS Exception handling diff --git a/demo/index.js b/demo/index.js index 0218166e..e6ee6dd8 100644 --- a/demo/index.js +++ b/demo/index.js @@ -492,15 +492,13 @@ function setupMenu() { menu.process = new Menu(document.body, '', { top, left: x[2] }); menu.process.addList('backend', ['cpu', 'webgl', 'wasm', 'humangl'], human.config.backend, (val) => human.config.backend = val); menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val); - // menu.process.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val); - // menu.process.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val); menu.process.addBool('use web worker', ui, 'useWorker'); menu.process.addHTML('
'); menu.process.addLabel('model parameters'); - menu.process.addRange('max objects', human.config.face.detector, 'maxFaces', 1, 50, 1, (val) => { - human.config.face.detector.maxFaces = parseInt(val); - human.config.body.maxDetections = parseInt(val); - human.config.hand.maxHands = parseInt(val); + menu.process.addRange('max objects', human.config.face.detector, 'maxDetected', 1, 50, 1, (val) => { + human.config.face.detector.maxDetected = parseInt(val); + human.config.body.maxDetected = parseInt(val); + human.config.hand.maxDetected = parseInt(val); }); menu.process.addRange('skip frames', human.config.face.detector, 'skipFrames', 0, 50, 1, (val) => { human.config.face.detector.skipFrames = parseInt(val); @@ -512,11 +510,6 @@ function setupMenu() { human.config.face.emotion.minConfidence = parseFloat(val); human.config.hand.minConfidence = parseFloat(val); }); - menu.process.addRange('score threshold', human.config.face.detector, 'scoreThreshold', 0.1, 1.0, 0.05, (val) => { - human.config.face.detector.scoreThreshold = parseFloat(val); - human.config.hand.scoreThreshold = parseFloat(val); - human.config.body.scoreThreshold = parseFloat(val); - }); menu.process.addRange('overlap', human.config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => { human.config.face.detector.iouThreshold = parseFloat(val); human.config.hand.iouThreshold = parseFloat(val); diff --git a/package.json b/package.json index aae746af..9ad425b7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@vladmandic/human", - "version": "1.7.1", + "version": "1.8.0", "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition", "sideEffects": false, "main": "dist/human.node.js", diff --git a/src/age/age.ts b/src/age/age.ts index b7ff470f..f4d6b360 100644 --- a/src/age/age.ts +++ b/src/age/age.ts @@ -1,6 +1,5 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import * as profile from '../profile'; let model; let last = { age: 0 }; @@ -31,14 +30,7 @@ export async function predict(image, config) { let ageT; const obj = { age: 0 }; - if (!config.profile) { - if (config.face.age.enabled) ageT = await model.predict(enhance); - } else { - const profileAge = config.face.age.enabled ? await tf.profile(() => model.predict(enhance)) : {}; - ageT = profileAge.result.clone(); - profileAge.result.dispose(); - profile.run('age', profileAge); - } + if (config.face.age.enabled) ageT = await model.predict(enhance); enhance.dispose(); if (ageT) { diff --git a/src/config.ts b/src/config.ts index 7c133c72..38f25c5e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -9,33 +9,34 @@ export interface Config { /** Backend used for TFJS operations */ backend: null | '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow', + /** Path to *.wasm files if backend is set to `wasm` */ wasmPath: string, + /** Print debug statements to console */ debug: boolean, + /** Perform model loading and inference concurrently or sequentially */ async: boolean, - /** Collect and print profiling data during inference operations */ - profile: boolean, - /** Internal: Use aggressive GPU memory deallocator when backend is set to `webgl` or `humangl` */ - deallocate: boolean, - /** Internal: Run all inference operations in an explicit local scope run to avoid memory leaks */ - scoped: boolean, + /** Perform additional optimizations when input is video, * - must be disabled for images * - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs * - skips boundary detection for every `skipFrames` frames specified for each model * - while maintaining in-box detection since objects don't change definition as fast */ videoOptimized: boolean, + /** What to use for `human.warmup()` * - warmup pre-initializes all models for faster inference but can take significant time on startup * - only used for `webgl` and `humangl` backends */ warmup: 'none' | 'face' | 'full' | 'body', + /** Base model path (typically starting with file://, http:// or https://) for all models - * - individual modelPath values are joined to this path + * - individual modelPath values are relative to this path */ modelBasePath: string, + /** Run input through image filters before inference * - image filters run with near-zero latency as they are executed on the GPU */ @@ -90,31 +91,30 @@ export interface Config { gesture: { enabled: boolean, }, + /** Controlls and configures all face-specific options: * - face detection, face mesh detection, age, gender, emotion detection and face description * Parameters: * - enabled: true/false - * - modelPath: path for individual face model + * - modelPath: path for each of face models + * - minConfidence: threshold for discarding a prediction + * - iouThreshold: ammount of overlap between two detected objects before one object is removed + * - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance * - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles - * - maxFaces: maximum number of faces detected in the input, should be set to the minimum number for performance * - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true * - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle - * - minConfidence: threshold for discarding a prediction - * - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression - * - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression - * - return extracted face as tensor for futher user processing + * - return: return extracted face as tensor for futher user processing */ face: { enabled: boolean, detector: { modelPath: string, rotation: boolean, - maxFaces: number, + maxDetected: number, skipFrames: number, skipInitial: boolean, minConfidence: number, iouThreshold: number, - scoreThreshold: number, return: boolean, }, mesh: { @@ -138,31 +138,30 @@ export interface Config { modelPath: string, }, }, + /** Controlls and configures all body detection specific options * - enabled: true/false - * - modelPath: paths for both hand detector model and hand skeleton model - * - maxDetections: maximum number of people detected in the input, should be set to the minimum number for performance - * - scoreThreshold: threshold for deciding when to remove people based on score in non-maximum suppression - * - nmsRadius: threshold for deciding whether body parts overlap too much in non-maximum suppression + * - modelPath: body pose model, can be absolute path or relative to modelBasePath + * - minConfidence: threshold for discarding a prediction + * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance */ body: { enabled: boolean, modelPath: string, - maxDetections: number, - scoreThreshold: number, - nmsRadius: number, + maxDetected: number, + minConfidence: number, }, + /** Controlls and configures all hand detection specific options * - enabled: true/false - * - modelPath: paths for both hand detector model and hand skeleton model + * - landmarks: detect hand landmarks or just hand boundary box + * - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath + * - minConfidence: threshold for discarding a prediction + * - iouThreshold: ammount of overlap between two detected objects before one object is removed + * - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted * - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true * - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle - * - minConfidence: threshold for discarding a prediction - * - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression - * - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression - * - maxHands: maximum number of hands detected in the input, should be set to the minimum number for performance - * - landmarks: detect hand landmarks or just hand boundary box */ hand: { enabled: boolean, @@ -171,8 +170,7 @@ export interface Config { skipInitial: boolean, minConfidence: number, iouThreshold: number, - scoreThreshold: number, - maxHands: number, + maxDetected: number, landmarks: boolean, detector: { modelPath: string, @@ -181,10 +179,13 @@ export interface Config { modelPath: string, }, }, + /** Controlls and configures all object detection specific options + * - enabled: true/false + * - modelPath: object detection model, can be absolute path or relative to modelBasePath * - minConfidence: minimum score that detection must have to return as valid object * - iouThreshold: ammount of overlap between two detected objects before one object is removed - * - maxResults: maximum number of detections to return + * - maxDetected: maximum number of detections to return * - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true */ object: { @@ -192,40 +193,20 @@ export interface Config { modelPath: string, minConfidence: number, iouThreshold: number, - maxResults: number, + maxDetected: number, skipFrames: number, }, } const config: Config = { - backend: 'webgl', // select tfjs backend to use + backend: 'webgl', // select tfjs backend to use, leave empty to use default backend // can be 'webgl', 'wasm', 'cpu', or 'humangl' which is a custom version of webgl - // leave as empty string to continue using default backend - // when backend is set outside of Human library modelBasePath: '../models/', // base path for all models - wasmPath: '../assets/', // path for wasm binaries - // only used for backend: wasm + wasmPath: '../assets/', // path for wasm binariesm, only used for backend: wasm debug: true, // print additional status messages to console async: true, // execute enabled models in parallel - // this disables per-model performance data but - // slightly increases performance - // cannot be used if profiling is enabled - profile: false, // internal: enable tfjs profiling - // this has significant performance impact - // only enable for debugging purposes - // currently only implemented for age,gender,emotion models - deallocate: false, // internal: aggresively deallocate gpu memory after each usage - // only valid for webgl and humangl backend and only during first call - // cannot be changed unless library is reloaded - // this has significant performance impact - // only enable on low-memory devices - scoped: false, // internal: enable scoped runs - // some models *may* have memory leaks, - // this wrapps everything in a local scope at a cost of performance - // typically not needed videoOptimized: true, // perform additional optimizations when input is video, - // must be disabled for images - // automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs + // automatically disabled for Image, ImageData, ImageBitmap // skips boundary detection for every n frames // while maintaining in-box detection since objects cannot move that fast warmup: 'face', // what to use for human.warmup(), can be 'none', 'face', 'full' @@ -258,7 +239,7 @@ const config: Config = { }, gesture: { - enabled: true, // enable simple gesture recognition + enabled: true, // enable gesture recognition based on model results }, face: { @@ -267,12 +248,11 @@ const config: Config = { // detector, mesh, iris, age, gender, emotion // (note: module is not loaded until it is required) detector: { - modelPath: 'blazeface-back.json', // detector model - // can be either absolute path or relative to modelBasePath + modelPath: 'blazeface-back.json', // detector model, can be absolute path or relative to modelBasePath rotation: false, // use best-guess rotated face image or just box with rotation as-is // false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees // this parameter is not valid in nodejs - maxFaces: 10, // maximum number of faces detected in the input + maxDetected: 10, // maximum number of faces detected in the input // should be set to the minimum number for performance skipFrames: 21, // how many frames to go without re-running the face bounding box detector // only used for video inputs @@ -282,18 +262,13 @@ const config: Config = { skipInitial: false, // if previous detection resulted in no faces detected, // should skipFrames be reset immediately to force new detection cycle minConfidence: 0.2, // threshold for discarding a prediction - iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in - // non-maximum suppression (0.1 means drop if overlap 10%) - scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score - // in non-maximum suppression, - // this is applied on detection objects only and before minConfidence + iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed return: false, // return extracted face as tensor }, mesh: { enabled: true, - modelPath: 'facemesh.json', // facemesh model - // can be either absolute path or relative to modelBasePath + modelPath: 'facemesh.json', // facemesh model, can be absolute path or relative to modelBasePath }, iris: { @@ -316,25 +291,18 @@ const config: Config = { enabled: true, minConfidence: 0.1, // threshold for discarding a prediction skipFrames: 32, // how many frames to go without re-running the detector - modelPath: 'emotion.json', // face emotion model - // can be either absolute path or relative to modelBasePath + modelPath: 'emotion.json', // face emotion model, can be absolute path or relative to modelBasePath }, }, body: { enabled: true, - modelPath: 'posenet.json', // body model - // can be either absolute path or relative to modelBasePath - // can be 'posenet', 'blazepose' or 'efficientpose' - // 'blazepose' and 'efficientpose' are experimental - maxDetections: 1, // maximum number of people detected in the input + modelPath: 'posenet.json', // body model, can be absolute path or relative to modelBasePath + // can be 'posenet' or 'blazepose' + maxDetected: 1, // maximum number of people detected in the input // should be set to the minimum number for performance // only valid for posenet as blazepose only detects single pose - scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score - // in non-maximum suppression - // only valid for posenet as blazepose only detects single pose - nmsRadius: 20, // radius for deciding points are too close in non-maximum suppression - // only valid for posenet as blazepose only detects single pose + minConfidence: 0.2, // threshold for discarding a prediction }, hand: { @@ -349,32 +317,24 @@ const config: Config = { skipInitial: false, // if previous detection resulted in no hands detected, // should skipFrames be reset immediately to force new detection cycle minConfidence: 0.1, // threshold for discarding a prediction - iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much - // in non-maximum suppression - scoreThreshold: 0.5, // threshold for deciding when to remove boxes based on - // score in non-maximum suppression - maxHands: 1, // maximum number of hands detected in the input + iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed + maxDetected: 1, // maximum number of hands detected in the input // should be set to the minimum number for performance landmarks: true, // detect hand landmarks or just hand boundary box detector: { - modelPath: 'handdetect.json', // hand detector model - // can be either absolute path or relative to modelBasePath + modelPath: 'handdetect.json', // hand detector model, can be absolute path or relative to modelBasePath }, skeleton: { - modelPath: 'handskeleton.json', // hand skeleton model - // can be either absolute path or relative to modelBasePath + modelPath: 'handskeleton.json', // hand skeleton model, can be absolute path or relative to modelBasePath }, }, object: { enabled: false, - modelPath: 'nanodet.json', // object detection model - // can be either absolute path or relative to modelBasePath - // 'nanodet' is experimental - minConfidence: 0.20, // threshold for discarding a prediction - iouThreshold: 0.40, // threshold for deciding whether boxes overlap too much - // in non-maximum suppression - maxResults: 10, // maximum number of objects detected in the input + modelPath: 'nanodet.json', // experimental: object detection model, can be absolute path or relative to modelBasePath + minConfidence: 0.2, // threshold for discarding a prediction + iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed + maxDetected: 10, // maximum number of objects detected in the input skipFrames: 41, // how many frames to go without re-running the detector }, }; diff --git a/src/draw/draw.ts b/src/draw/draw.ts index f781e2ef..a016c19e 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -294,68 +294,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, draw // shoulder line points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // torso main points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightHip'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftHip'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso // leg left points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftHip'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftKnee'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftHeel'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftFoot'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // leg right points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightHip'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightKnee'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightHeel'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightFoot'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // arm left points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftElbow'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftWrist'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftPalm'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // arm right points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightElbow'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightWrist'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightPalm'); - if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); + if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // draw all } diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index dad57777..6f675378 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -1,6 +1,5 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import * as profile from '../profile'; let model; let keypoints: Array = []; @@ -55,15 +54,7 @@ export async function predict(image, config) { }); let resT; - - if (!config.profile) { - if (config.body.enabled) resT = await model.predict(tensor); - } else { - const profileT = config.body.enabled ? await tf.profile(() => model.predict(tensor)) : {}; - resT = profileT.result.clone(); - profileT.result.dispose(); - profile.run('body', profileT); - } + if (config.body.enabled) resT = await model.predict(tensor); tensor.dispose(); if (resT) { @@ -76,8 +67,8 @@ export async function predict(image, config) { // process each unstacked tensor as a separate body part for (let id = 0; id < stack.length; id++) { // actual processing to get coordinates and score - const [x, y, score] = max2d(stack[id], config.body.scoreThreshold); - if (score > config.body.scoreThreshold) { + const [x, y, score] = max2d(stack[id], config.body.minConfidence); + if (score > config.body.minConfidence) { parts.push({ id, score: Math.round(100 * score) / 100, diff --git a/src/emotion/emotion.ts b/src/emotion/emotion.ts index dc52d502..2e1c8910 100644 --- a/src/emotion/emotion.ts +++ b/src/emotion/emotion.ts @@ -1,6 +1,5 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import * as profile from '../profile'; const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']; let model; @@ -46,17 +45,9 @@ export async function predict(image, config) { grayscale.dispose(); const obj: Array<{ score: number, emotion: string }> = []; if (config.face.emotion.enabled) { - let data; - if (!config.profile) { - const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation - data = emotionT.dataSync(); - tf.dispose(emotionT); - } else { - const profileData = await tf.profile(() => model.predict(normalize)); - data = profileData.result.dataSync(); - profileData.result.dispose(); - profile.run('emotion', profileData); - } + const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation + const data = emotionT.dataSync(); + tf.dispose(emotionT); for (let i = 0; i < data.length; i++) { if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] }); } diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index f583bf32..e0f8ba43 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -1,6 +1,5 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import * as profile from '../profile'; let model; let last = { age: 0 }; @@ -108,13 +107,7 @@ export async function predict(image, config) { genderConfidence: 0, descriptor: [] }; - if (!config.profile) { - if (config.face.description.enabled) resT = await model.predict(enhanced); - } else { - const profileDesc = config.face.description.enabled ? await tf.profile(() => model.predict(enhanced)) : {}; - resT = profileDesc.result; - profile.run('faceres', profileDesc); - } + if (config.face.description.enabled) resT = await model.predict(enhanced); tf.dispose(enhanced); if (resT) { diff --git a/src/gender/gender.ts b/src/gender/gender.ts index 511055d5..f8d6d145 100644 --- a/src/gender/gender.ts +++ b/src/gender/gender.ts @@ -1,6 +1,5 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import * as profile from '../profile'; let model; let last = { gender: '' }; @@ -49,14 +48,7 @@ export async function predict(image, config) { let genderT; const obj = { gender: '', confidence: 0 }; - if (!config.profile) { - if (config.face.gender.enabled) genderT = await model.predict(enhance); - } else { - const profileGender = config.face.gender.enabled ? await tf.profile(() => model.predict(enhance)) : {}; - genderT = profileGender.result.clone(); - profileGender.result.dispose(); - profile.run('gender', profileGender); - } + if (config.face.gender.enabled) genderT = await model.predict(enhance); enhance.dispose(); if (genderT) { diff --git a/src/handpose/handdetector.ts b/src/handpose/handdetector.ts index 2d861b2d..a4ab76df 100644 --- a/src/handpose/handdetector.ts +++ b/src/handpose/handdetector.ts @@ -46,7 +46,7 @@ export class HandDetector { const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]); const boxes = this.normalizeBoxes(rawBoxes); rawBoxes.dispose(); - const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxHands, config.hand.iouThreshold, config.hand.scoreThreshold); + const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence); const filtered = filteredT.arraySync(); scoresT.dispose(); diff --git a/src/handpose/handpipeline.ts b/src/handpose/handpipeline.ts index d6ba37e4..20c5541e 100644 --- a/src/handpose/handpipeline.ts +++ b/src/handpose/handpipeline.ts @@ -83,7 +83,7 @@ export class HandPipeline { if (config.videoOptimized) this.skipped++; // if detector result count doesn't match current working set, use it to reset current working set - if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxHands) || !config.hand.landmarks)) { + if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxDetected) || !config.hand.landmarks)) { this.detectedHands = 0; this.storedBoxes = [...boxes]; // for (const possible of boxes) this.storedBoxes.push(possible); diff --git a/src/human.ts b/src/human.ts index b5db9c67..32c46f23 100644 --- a/src/human.ts +++ b/src/human.ts @@ -13,7 +13,6 @@ import * as nanodet from './nanodet/nanodet'; import * as gesture from './gesture/gesture'; import * as image from './image/image'; import * as draw from './draw/draw'; -import * as profile from './profile'; import { Config, defaults } from './config'; import { Result } from './result'; import * as sample from './sample'; @@ -168,14 +167,6 @@ export class Human { this.sysinfo = sysinfo.info(); } - /** Internal: ProfileData method returns last known profiling information - * - Requires human.config.profile set to true - */ - profileData(): { newBytes, newTensors, peakBytes, numKernelOps, timeKernelOps, slowestKernelOps, largestKernelOps } | {} { - if (this.config.profile) return profile.data; - return {}; - } - // helper function: measure tensor leak /** @hidden */ analyze = (...msg) => { @@ -335,9 +326,9 @@ export class Human { if (this.tf.getBackend() === 'webgl' || this.tf.getBackend() === 'humangl') { this.tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false); this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true); - if (this.config.deallocate) { - log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate); - this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1); + if (typeof this.config['deallocate'] !== 'undefined') { + log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true); + this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', 0); } const gl = await this.tf.backend().getGPGPUContext().gl; if (this.config.debug) log(`gl version:${gl.getParameter(gl.VERSION)} renderer:${gl.getParameter(gl.RENDERER)}`); @@ -378,9 +369,6 @@ export class Human { // load models if enabled await this.load(); - if (this.config.scoped) this.tf.engine().startScope(); - this.analyze('Start Scope:'); - // disable video optimization for inputs of type image, but skip if inside worker thread let previousVideoOptimized; // @ts-ignore ignore missing type for WorkerGlobalScope as that is the point @@ -474,9 +462,6 @@ export class Human { } tf.dispose(process.tensor); - if (this.config.scoped) this.tf.engine().endScope(); - this.analyze('End Scope:'); - // run gesture analysis last let gestureRes: any[] = []; if (this.config.gesture.enabled) { diff --git a/src/nanodet/nanodet.ts b/src/nanodet/nanodet.ts index b2910ff8..08da16b7 100644 --- a/src/nanodet/nanodet.ts +++ b/src/nanodet/nanodet.ts @@ -1,6 +1,5 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import * as profile from '../profile'; import { labels } from './labels'; let model; @@ -83,7 +82,7 @@ async function process(res, inputSize, outputShape, config) { const nmsScores = results.map((a) => a.score); let nmsIdx: any[] = []; if (nmsBoxes && nmsBoxes.length > 0) { - const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxResults, config.object.iouThreshold, config.object.minConfidence); + const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence); nmsIdx = nms.dataSync(); tf.dispose(nms); } @@ -114,13 +113,7 @@ export async function predict(image, config) { resize.dispose(); let objectT; - if (!config.profile) { - if (config.object.enabled) objectT = await model.predict(transpose); - } else { - const profileObject = config.object.enabled ? await tf.profile(() => model.predict(transpose)) : {}; - objectT = profileObject.result; - profile.run('object', profileObject); - } + if (config.object.enabled) objectT = await model.predict(transpose); transpose.dispose(); const obj = await process(objectT, model.inputSize, outputSize, config); diff --git a/src/posenet/poses.ts b/src/posenet/poses.ts index f454cf5a..2d8cee59 100644 --- a/src/posenet/poses.ts +++ b/src/posenet/poses.ts @@ -3,6 +3,7 @@ import * as kpt from './keypoints'; const localMaximumRadius = 1; const defaultOutputStride = 16; +const squaredNmsRadius = 20 ** 2; function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) { const getDisplacement = (point) => ({ @@ -86,7 +87,7 @@ function scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scor return localMaximum; } -export function buildPartWithScoreQueue(scoreThreshold, scores) { +export function buildPartWithScoreQueue(minConfidence, scores) { const [height, width, numKeypoints] = scores.shape; const queue = new utils.MaxHeap(height * width * numKeypoints, ({ score }) => score); for (let heatmapY = 0; heatmapY < height; ++heatmapY) { @@ -94,7 +95,7 @@ export function buildPartWithScoreQueue(scoreThreshold, scores) { for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) { const score = scores.get(heatmapY, heatmapX, keypointId); // Only consider parts with score greater or equal to threshold as root candidates. - if (score < scoreThreshold) continue; + if (score < minConfidence) continue; // Only consider keypoints whose score is maximum in a local window. if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scores)) queue.enqueue({ score, part: { heatmapY, heatmapX, id: keypointId } }); } @@ -103,38 +104,37 @@ export function buildPartWithScoreQueue(scoreThreshold, scores) { return queue; } -function withinRadius(poses, squaredNmsRadius, { x, y }, keypointId) { +function withinRadius(poses, { x, y }, keypointId) { return poses.some(({ keypoints }) => { const correspondingKeypoint = keypoints[keypointId].position; return utils.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= squaredNmsRadius; }); } -function getInstanceScore(existingPoses, squaredNmsRadius, instanceKeypoints) { +function getInstanceScore(existingPoses, instanceKeypoints) { const notOverlappedKeypointScores = instanceKeypoints.reduce((result, { position, score }, keypointId) => { - if (!withinRadius(existingPoses, squaredNmsRadius, position, keypointId)) result += score; + if (!withinRadius(existingPoses, position, keypointId)) result += score; return result; }, 0.0); return notOverlappedKeypointScores / instanceKeypoints.length; } -export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, nmsRadius, maxDetections, scoreThreshold) { +export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, maxDetected, minConfidence) { const poses: Array<{ keypoints: any, box: any, score: number }> = []; - const queue = buildPartWithScoreQueue(scoreThreshold, scoresBuffer); - const squaredNmsRadius = nmsRadius ** 2; - // Generate at most maxDetections object instances per image in decreasing root part score order. - while (poses.length < maxDetections && !queue.empty()) { + const queue = buildPartWithScoreQueue(minConfidence, scoresBuffer); + // Generate at most maxDetected object instances per image in decreasing root part score order. + while (poses.length < maxDetected && !queue.empty()) { // The top element in the queue is the next root candidate. const root = queue.dequeue(); // Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance. const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer); - if (withinRadius(poses, squaredNmsRadius, rootImageCoords, root.part.id)) continue; + if (withinRadius(poses, rootImageCoords, root.part.id)) continue; // Else start a new detection instance at the position of the root. const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer); - const keypoints = allKeypoints.filter((a) => a.score > scoreThreshold); - const score = getInstanceScore(poses, squaredNmsRadius, keypoints); + const keypoints = allKeypoints.filter((a) => a.score > minConfidence); + const score = getInstanceScore(poses, keypoints); const box = utils.getBoundingBox(keypoints); - if (score > scoreThreshold) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 }); + if (score > minConfidence) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 }); } return poses; } diff --git a/src/profile.ts b/src/profile.ts index cd9101fa..7390058a 100644 --- a/src/profile.ts +++ b/src/profile.ts @@ -4,7 +4,7 @@ export const data = {}; export function run(modelName: string, profileData: any): void { if (!profileData || !profileData.kernels) return; - const maxResults = 5; + const maxDetected = 5; const time = profileData.kernels .filter((a) => a.kernelTimeMs > 0) .reduce((a, b) => a += b.kernelTimeMs, 0); @@ -16,8 +16,8 @@ export function run(modelName: string, profileData: any): void { .map((a, i) => { a.id = i; return a; }) .filter((a) => a.totalBytesSnapshot > 0) .sort((a, b) => b.totalBytesSnapshot - a.totalBytesSnapshot); - if (slowest.length > maxResults) slowest.length = maxResults; - if (largest.length > maxResults) largest.length = maxResults; + if (slowest.length > maxDetected) slowest.length = maxDetected; + if (largest.length > maxDetected) largest.length = maxDetected; data[modelName] = { model: modelName, newBytes: profileData.newBytes, diff --git a/wiki b/wiki index 3b81af15..90624448 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 3b81af15f2560de5c06f20cbd8de57caf62682f2 +Subproject commit 906244487754b61fd24f49fe2db91ea68264137d