major update for 1.8 release candidate

CHANGELOG.md

@@ -1,6 +1,6 @@
 # @vladmandic/human  
 
-Version: **1.7.1**  
+Version: **1.8.0**  
 Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**  
 
 Author: **Vladimir Mandic <mandic00@live.com>**  
@@ -9,11 +9,12 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
+### **HEAD -> main** 2021/04/25 mandic00@live.com
+
+
 ### **1.7.1** 2021/04/25 mandic00@live.com
 
-
+- remove obsolete binary models
-### **origin/main** 2021/04/24 mandic00@live.com
-
 - enable cross origin isolation
 - rewrite posenet decoder
 - remove efficientpose

TODO.md

@@ -17,4 +17,21 @@ N/A
 - Blazepose  
   Needs detector before running pose to center the image
 
-## Soon to be Removed
+## RC: 1.8
+
+### Done
+
+Major configuration simplification:
+
+- Unified minConfidence and scoreThresdold as minConfidence
+- Replaced nmsRadius with built-in default
+- Replaced maxFaces, maxDetections, maxHands, maxResults with maxDetected
+- Remove deallocate, profile, scoped
+
+Stop building sourcemaps for NodeJS deliverables
+
+### TBD
+
+- Remove modelPaths
+- Remove blazeface-front, replace blazeface-back with blazeface
+- NodeJS Exception handling

demo/index.js

@@ -492,15 +492,13 @@ function setupMenu() {
   menu.process = new Menu(document.body, '', { top, left: x[2] });
   menu.process.addList('backend', ['cpu', 'webgl', 'wasm', 'humangl'], human.config.backend, (val) => human.config.backend = val);
   menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val);
-  // menu.process.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val);
-  // menu.process.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val);
   menu.process.addBool('use web worker', ui, 'useWorker');
   menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">');
   menu.process.addLabel('model parameters');
-  menu.process.addRange('max objects', human.config.face.detector, 'maxFaces', 1, 50, 1, (val) => {
+  menu.process.addRange('max objects', human.config.face.detector, 'maxDetected', 1, 50, 1, (val) => {
-    human.config.face.detector.maxFaces = parseInt(val);
+    human.config.face.detector.maxDetected = parseInt(val);
-    human.config.body.maxDetections = parseInt(val);
+    human.config.body.maxDetected = parseInt(val);
-    human.config.hand.maxHands = parseInt(val);
+    human.config.hand.maxDetected = parseInt(val);
   });
   menu.process.addRange('skip frames', human.config.face.detector, 'skipFrames', 0, 50, 1, (val) => {
     human.config.face.detector.skipFrames = parseInt(val);
@@ -512,11 +510,6 @@ function setupMenu() {
     human.config.face.emotion.minConfidence = parseFloat(val);
     human.config.hand.minConfidence = parseFloat(val);
   });
-  menu.process.addRange('score threshold', human.config.face.detector, 'scoreThreshold', 0.1, 1.0, 0.05, (val) => {
-    human.config.face.detector.scoreThreshold = parseFloat(val);
-    human.config.hand.scoreThreshold = parseFloat(val);
-    human.config.body.scoreThreshold = parseFloat(val);
-  });
   menu.process.addRange('overlap', human.config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => {
     human.config.face.detector.iouThreshold = parseFloat(val);
     human.config.hand.iouThreshold = parseFloat(val);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vladmandic/human",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
   "sideEffects": false,
   "main": "dist/human.node.js",

src/age/age.ts

@@ -1,6 +1,5 @@
 import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as profile from '../profile';
 
 let model;
 let last = { age: 0 };
@@ -31,14 +30,7 @@ export async function predict(image, config) {
     let ageT;
     const obj = { age: 0 };
 
-    if (!config.profile) {
     if (config.face.age.enabled) ageT = await model.predict(enhance);
-    } else {
-      const profileAge = config.face.age.enabled ? await tf.profile(() => model.predict(enhance)) : {};
-      ageT = profileAge.result.clone();
-      profileAge.result.dispose();
-      profile.run('age', profileAge);
-    }
     enhance.dispose();
 
     if (ageT) {

src/config.ts

@@ -9,33 +9,34 @@
 export interface Config {
   /** Backend used for TFJS operations */
   backend: null | '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow',
+
   /** Path to *.wasm files if backend is set to `wasm` */
   wasmPath: string,
+
   /** Print debug statements to console */
   debug: boolean,
+
   /** Perform model loading and inference concurrently or sequentially */
   async: boolean,
-  /** Collect and print profiling data during inference operations */
+
-  profile: boolean,
-  /** Internal: Use aggressive GPU memory deallocator when backend is set to `webgl` or `humangl` */
-  deallocate: boolean,
-  /** Internal: Run all inference operations in an explicit local scope run to avoid memory leaks */
-  scoped: boolean,
   /** Perform additional optimizations when input is video,
    * - must be disabled for images
    * - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs
    * - skips boundary detection for every `skipFrames` frames specified for each model
    * - while maintaining in-box detection since objects don't change definition as fast */
   videoOptimized: boolean,
+
   /** What to use for `human.warmup()`
    * - warmup pre-initializes all models for faster inference but can take significant time on startup
    * - only used for `webgl` and `humangl` backends
   */
   warmup: 'none' | 'face' | 'full' | 'body',
+
   /** Base model path (typically starting with file://, http:// or https://) for all models
-   * - individual modelPath values are joined to this path
+   * - individual modelPath values are relative to this path
   */
   modelBasePath: string,
+
   /** Run input through image filters before inference
    * - image filters run with near-zero latency as they are executed on the GPU
   */
@@ -90,31 +91,30 @@ export interface Config {
   gesture: {
     enabled: boolean,
   },
+
   /** Controlls and configures all face-specific options:
    * - face detection, face mesh detection, age, gender, emotion detection and face description
    * Parameters:
    * - enabled: true/false
-   * - modelPath: path for individual face model
+   * - modelPath: path for each of face models
+   * - minConfidence: threshold for discarding a prediction
+   * - iouThreshold: ammount of overlap between two detected objects before one object is removed
+   * - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance
    * - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles
-   * - maxFaces: maximum number of faces detected in the input, should be set to the minimum number for performance
    * - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true
    * - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle
-   * - minConfidence: threshold for discarding a prediction
+   * - return: return extracted face as tensor for futher user processing
-   * - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
-   * - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
-   * - return extracted face as tensor for futher user processing
   */
   face: {
     enabled: boolean,
     detector: {
       modelPath: string,
       rotation: boolean,
-      maxFaces: number,
+      maxDetected: number,
       skipFrames: number,
       skipInitial: boolean,
       minConfidence: number,
       iouThreshold: number,
-      scoreThreshold: number,
       return: boolean,
     },
     mesh: {
@@ -138,31 +138,30 @@ export interface Config {
       modelPath: string,
     },
   },
+
   /** Controlls and configures all body detection specific options
    * - enabled: true/false
-   * - modelPath: paths for both hand detector model and hand skeleton model
+   * - modelPath: body pose model, can be absolute path or relative to modelBasePath
-   * - maxDetections: maximum number of people detected in the input, should be set to the minimum number for performance
+   * - minConfidence: threshold for discarding a prediction
-   * - scoreThreshold: threshold for deciding when to remove people based on score in non-maximum suppression
+   * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance
-   * - nmsRadius: threshold for deciding whether body parts overlap too much in non-maximum suppression
   */
   body: {
     enabled: boolean,
     modelPath: string,
-    maxDetections: number,
+    maxDetected: number,
-    scoreThreshold: number,
+    minConfidence: number,
-    nmsRadius: number,
   },
+
   /** Controlls and configures all hand detection specific options
    * - enabled: true/false
-   * - modelPath: paths for both hand detector model and hand skeleton model
+   * - landmarks: detect hand landmarks or just hand boundary box
+   * - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath
+   * - minConfidence: threshold for discarding a prediction
+   * - iouThreshold: ammount of overlap between two detected objects before one object is removed
+   * - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance
    * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
    * - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true
    * - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle
-   * - minConfidence: threshold for discarding a prediction
-   * - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
-   * - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
-   * - maxHands: maximum number of hands detected in the input, should be set to the minimum number for performance
-   * - landmarks: detect hand landmarks or just hand boundary box
   */
   hand: {
     enabled: boolean,
@@ -171,8 +170,7 @@ export interface Config {
     skipInitial: boolean,
     minConfidence: number,
     iouThreshold: number,
-    scoreThreshold: number,
+    maxDetected: number,
-    maxHands: number,
     landmarks: boolean,
     detector: {
       modelPath: string,
@@ -181,10 +179,13 @@ export interface Config {
       modelPath: string,
     },
   },
+
   /** Controlls and configures all object detection specific options
+   * - enabled: true/false
+   * - modelPath: object detection model, can be absolute path or relative to modelBasePath
    * - minConfidence: minimum score that detection must have to return as valid object
    * - iouThreshold: ammount of overlap between two detected objects before one object is removed
-   * - maxResults: maximum number of detections to return
+   * - maxDetected: maximum number of detections to return
    * - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true
   */
   object: {
@@ -192,40 +193,20 @@ export interface Config {
     modelPath: string,
     minConfidence: number,
     iouThreshold: number,
-    maxResults: number,
+    maxDetected: number,
     skipFrames: number,
   },
 }
 
 const config: Config = {
-  backend: 'webgl',          // select tfjs backend to use
+  backend: 'webgl',          // select tfjs backend to use, leave empty to use default backend
                              // can be 'webgl', 'wasm', 'cpu', or 'humangl' which is a custom version of webgl
-                             // leave as empty string to continue using default backend
-                             // when backend is set outside of Human library
   modelBasePath: '../models/', // base path for all models
-  wasmPath: '../assets/',    // path for wasm binaries
+  wasmPath: '../assets/',    // path for wasm binariesm, only used for backend: wasm
-                             // only used for backend: wasm
   debug: true,               // print additional status messages to console
   async: true,               // execute enabled models in parallel
-                             // this disables per-model performance data but
-                             // slightly increases performance
-                             // cannot be used if profiling is enabled
-  profile: false,            // internal: enable tfjs profiling
-                             // this has significant performance impact
-                             // only enable for debugging purposes
-                             // currently only implemented for age,gender,emotion models
-  deallocate: false,         // internal: aggresively deallocate gpu memory after each usage
-                             // only valid for webgl and humangl backend and only during first call
-                             // cannot be changed unless library is reloaded
-                             // this has significant performance impact
-                             // only enable on low-memory devices
-  scoped: false,             // internal: enable scoped runs
-                             // some models *may* have memory leaks,
-                             // this wrapps everything in a local scope at a cost of performance
-                             // typically not needed
   videoOptimized: true,      // perform additional optimizations when input is video,
-                             // must be disabled for images
+                             // automatically disabled for Image, ImageData, ImageBitmap
-                             // automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs
                              // skips boundary detection for every n frames
                              // while maintaining in-box detection since objects cannot move that fast
   warmup: 'face',            // what to use for human.warmup(), can be 'none', 'face', 'full'
@@ -258,7 +239,7 @@ const config: Config = {
   },
 
   gesture: {
-    enabled: true,           // enable simple gesture recognition
+    enabled: true,           // enable gesture recognition based on model results
   },
 
   face: {
@@ -267,12 +248,11 @@ const config: Config = {
                              // detector, mesh, iris, age, gender, emotion
                              // (note: module is not loaded until it is required)
     detector: {
-      modelPath: 'blazeface-back.json', // detector model
+      modelPath: 'blazeface-back.json', // detector model, can be absolute path or relative to modelBasePath
-                             // can be either absolute path or relative to modelBasePath
       rotation: false,       // use best-guess rotated face image or just box with rotation as-is
                              // false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees
                              // this parameter is not valid in nodejs
-      maxFaces: 10,          // maximum number of faces detected in the input
+      maxDetected: 10,          // maximum number of faces detected in the input
                              // should be set to the minimum number for performance
       skipFrames: 21,        // how many frames to go without re-running the face bounding box detector
                              // only used for video inputs
@@ -282,18 +262,13 @@ const config: Config = {
       skipInitial: false,    // if previous detection resulted in no faces detected,
                              // should skipFrames be reset immediately to force new detection cycle
       minConfidence: 0.2,    // threshold for discarding a prediction
-      iouThreshold: 0.1,     // threshold for deciding whether boxes overlap too much in
+      iouThreshold: 0.1,     // ammount of overlap between two detected objects before one object is removed
-                             // non-maximum suppression (0.1 means drop if overlap 10%)
-      scoreThreshold: 0.2,   // threshold for deciding when to remove boxes based on score
-                             // in non-maximum suppression,
-                             // this is applied on detection objects only and before minConfidence
       return: false,         // return extracted face as tensor
     },
 
     mesh: {
       enabled: true,
-      modelPath: 'facemesh.json',  // facemesh model
+      modelPath: 'facemesh.json',  // facemesh model, can be absolute path or relative to modelBasePath
-                             // can be either absolute path or relative to modelBasePath
     },
 
     iris: {
@@ -316,25 +291,18 @@ const config: Config = {
       enabled: true,
       minConfidence: 0.1,    // threshold for discarding a prediction
       skipFrames: 32,        // how many frames to go without re-running the detector
-      modelPath: 'emotion.json',  // face emotion model
+      modelPath: 'emotion.json',  // face emotion model, can be absolute path or relative to modelBasePath
-                             // can be either absolute path or relative to modelBasePath
     },
   },
 
   body: {
     enabled: true,
-    modelPath: 'posenet.json',  // body model
+    modelPath: 'posenet.json',  // body model, can be absolute path or relative to modelBasePath
-                             // can be either absolute path or relative to modelBasePath
+                             // can be 'posenet' or 'blazepose'
-                             // can be 'posenet', 'blazepose' or 'efficientpose'
+    maxDetected: 1,          // maximum number of people detected in the input
-                             // 'blazepose' and 'efficientpose' are experimental
-    maxDetections: 1,        // maximum number of people detected in the input
                              // should be set to the minimum number for performance
                              // only valid for posenet as blazepose only detects single pose
-    scoreThreshold: 0.2,     // threshold for deciding when to remove boxes based on score
+    minConfidence: 0.2,      // threshold for discarding a prediction
-                             // in non-maximum suppression
-                             // only valid for posenet as blazepose only detects single pose
-    nmsRadius: 20,           // radius for deciding points are too close in non-maximum suppression
-                             // only valid for posenet as blazepose only detects single pose
   },
 
   hand: {
@@ -349,32 +317,24 @@ const config: Config = {
     skipInitial: false,      // if previous detection resulted in no hands detected,
                              // should skipFrames be reset immediately to force new detection cycle
     minConfidence: 0.1,      // threshold for discarding a prediction
-    iouThreshold: 0.1,       // threshold for deciding whether boxes overlap too much
+    iouThreshold: 0.1,       // ammount of overlap between two detected objects before one object is removed
-                             // in non-maximum suppression
+    maxDetected: 1,          // maximum number of hands detected in the input
-    scoreThreshold: 0.5,     // threshold for deciding when to remove boxes based on
-                             // score in non-maximum suppression
-    maxHands: 1,             // maximum number of hands detected in the input
                              // should be set to the minimum number for performance
     landmarks: true,         // detect hand landmarks or just hand boundary box
     detector: {
-      modelPath: 'handdetect.json',  // hand detector model
+      modelPath: 'handdetect.json',  // hand detector model, can be absolute path or relative to modelBasePath
-                             // can be either absolute path or relative to modelBasePath
     },
     skeleton: {
-      modelPath: 'handskeleton.json',  // hand skeleton model
+      modelPath: 'handskeleton.json',  // hand skeleton model, can be absolute path or relative to modelBasePath
-                             // can be either absolute path or relative to modelBasePath
     },
   },
 
   object: {
     enabled: false,
-    modelPath: 'nanodet.json',  // object detection model
+    modelPath: 'nanodet.json',  // experimental: object detection model, can be absolute path or relative to modelBasePath
-                             // can be either absolute path or relative to modelBasePath
+    minConfidence: 0.2,      // threshold for discarding a prediction
-                             // 'nanodet' is experimental
+    iouThreshold: 0.4,       // ammount of overlap between two detected objects before one object is removed
-    minConfidence: 0.20,     // threshold for discarding a prediction
+    maxDetected: 10,         // maximum number of objects detected in the input
-    iouThreshold: 0.40,      // threshold for deciding whether boxes overlap too much
-                             // in non-maximum suppression
-    maxResults: 10,          // maximum number of objects detected in the input
     skipFrames: 41,          // how many frames to go without re-running the detector
   },
 };

src/draw/draw.ts

@@ -294,68 +294,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, draw
       // shoulder line
       points.length = 0;
       part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       curves(ctx, points, localOptions);
       // torso main
       points.length = 0;
       part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'rightHip');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'leftHip');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
       // leg left
       points.length = 0;
       part = result[i].keypoints.find((a) => a.part === 'leftHip');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'leftKnee');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'leftHeel');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'leftFoot');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       curves(ctx, points, localOptions);
       // leg right
       points.length = 0;
       part = result[i].keypoints.find((a) => a.part === 'rightHip');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'rightKnee');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'rightHeel');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'rightFoot');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       curves(ctx, points, localOptions);
       // arm left
       points.length = 0;
       part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'leftElbow');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'leftWrist');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'leftPalm');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       curves(ctx, points, localOptions);
       // arm right
       points.length = 0;
       part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'rightElbow');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'rightWrist');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       part = result[i].keypoints.find((a) => a.part === 'rightPalm');
-      if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
+      if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
       curves(ctx, points, localOptions);
       // draw all
     }

src/efficientpose/efficientpose.ts

@@ -1,6 +1,5 @@
 import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as profile from '../profile';
 
 let model;
 let keypoints: Array<any> = [];
@@ -55,15 +54,7 @@ export async function predict(image, config) {
     });
 
     let resT;
-
-    if (!config.profile) {
     if (config.body.enabled) resT = await model.predict(tensor);
-    } else {
-      const profileT = config.body.enabled ? await tf.profile(() => model.predict(tensor)) : {};
-      resT = profileT.result.clone();
-      profileT.result.dispose();
-      profile.run('body', profileT);
-    }
     tensor.dispose();
 
     if (resT) {
@@ -76,8 +67,8 @@ export async function predict(image, config) {
       // process each unstacked tensor as a separate body part
       for (let id = 0; id < stack.length; id++) {
         // actual processing to get coordinates and score
-        const [x, y, score] = max2d(stack[id], config.body.scoreThreshold);
+        const [x, y, score] = max2d(stack[id], config.body.minConfidence);
-        if (score > config.body.scoreThreshold) {
+        if (score > config.body.minConfidence) {
           parts.push({
             id,
             score: Math.round(100 * score) / 100,

src/emotion/emotion.ts

@@ -1,6 +1,5 @@
 import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as profile from '../profile';
 
 const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'];
 let model;
@@ -46,17 +45,9 @@ export async function predict(image, config) {
     grayscale.dispose();
     const obj: Array<{ score: number, emotion: string }> = [];
     if (config.face.emotion.enabled) {
-      let data;
-      if (!config.profile) {
       const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation
-        data = emotionT.dataSync();
+      const data = emotionT.dataSync();
       tf.dispose(emotionT);
-      } else {
-        const profileData = await tf.profile(() => model.predict(normalize));
-        data = profileData.result.dataSync();
-        profileData.result.dispose();
-        profile.run('emotion', profileData);
-      }
       for (let i = 0; i < data.length; i++) {
         if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
       }

src/faceres/faceres.ts

@@ -1,6 +1,5 @@
 import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as profile from '../profile';
 
 let model;
 let last = { age: 0 };
@@ -108,13 +107,7 @@ export async function predict(image, config) {
       genderConfidence: <number>0,
       descriptor: <number[]>[] };
 
-    if (!config.profile) {
     if (config.face.description.enabled) resT = await model.predict(enhanced);
-    } else {
-      const profileDesc = config.face.description.enabled ? await tf.profile(() => model.predict(enhanced)) : {};
-      resT = profileDesc.result;
-      profile.run('faceres', profileDesc);
-    }
     tf.dispose(enhanced);
 
     if (resT) {

src/gender/gender.ts

@@ -1,6 +1,5 @@
 import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as profile from '../profile';
 
 let model;
 let last = { gender: '' };
@@ -49,14 +48,7 @@ export async function predict(image, config) {
     let genderT;
     const obj = { gender: '', confidence: 0 };
 
-    if (!config.profile) {
     if (config.face.gender.enabled) genderT = await model.predict(enhance);
-    } else {
-      const profileGender = config.face.gender.enabled ? await tf.profile(() => model.predict(enhance)) : {};
-      genderT = profileGender.result.clone();
-      profileGender.result.dispose();
-      profile.run('gender', profileGender);
-    }
     enhance.dispose();
 
     if (genderT) {

src/handpose/handdetector.ts

@@ -46,7 +46,7 @@ export class HandDetector {
     const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
     const boxes = this.normalizeBoxes(rawBoxes);
     rawBoxes.dispose();
-    const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxHands, config.hand.iouThreshold, config.hand.scoreThreshold);
+    const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
     const filtered = filteredT.arraySync();
 
     scoresT.dispose();

src/handpose/handpipeline.ts

@@ -83,7 +83,7 @@ export class HandPipeline {
     if (config.videoOptimized) this.skipped++;
 
     // if detector result count doesn't match current working set, use it to reset current working set
-    if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxHands) || !config.hand.landmarks)) {
+    if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxDetected) || !config.hand.landmarks)) {
       this.detectedHands = 0;
       this.storedBoxes = [...boxes];
       // for (const possible of boxes) this.storedBoxes.push(possible);

src/human.ts

@@ -13,7 +13,6 @@ import * as nanodet from './nanodet/nanodet';
 import * as gesture from './gesture/gesture';
 import * as image from './image/image';
 import * as draw from './draw/draw';
-import * as profile from './profile';
 import { Config, defaults } from './config';
 import { Result } from './result';
 import * as sample from './sample';
@@ -168,14 +167,6 @@ export class Human {
     this.sysinfo = sysinfo.info();
   }
 
-  /** Internal: ProfileData method returns last known profiling information
-   * - Requires human.config.profile set to true
-   */
-  profileData(): { newBytes, newTensors, peakBytes, numKernelOps, timeKernelOps, slowestKernelOps, largestKernelOps } | {} {
-    if (this.config.profile) return profile.data;
-    return {};
-  }
-
   // helper function: measure tensor leak
   /** @hidden */
   analyze = (...msg) => {
@@ -335,9 +326,9 @@ export class Human {
       if (this.tf.getBackend() === 'webgl' || this.tf.getBackend() === 'humangl') {
         this.tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false);
         this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
-        if (this.config.deallocate) {
+        if (typeof this.config['deallocate'] !== 'undefined') {
-          log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate);
+          log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true);
-          this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1);
+          this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', 0);
         }
         const gl = await this.tf.backend().getGPGPUContext().gl;
         if (this.config.debug) log(`gl version:${gl.getParameter(gl.VERSION)} renderer:${gl.getParameter(gl.RENDERER)}`);
@@ -378,9 +369,6 @@ export class Human {
       // load models if enabled
       await this.load();
 
-      if (this.config.scoped) this.tf.engine().startScope();
-      this.analyze('Start Scope:');
-
       // disable video optimization for inputs of type image, but skip if inside worker thread
       let previousVideoOptimized;
       // @ts-ignore ignore missing type for WorkerGlobalScope as that is the point
@@ -474,9 +462,6 @@ export class Human {
       }
       tf.dispose(process.tensor);
 
-      if (this.config.scoped) this.tf.engine().endScope();
-      this.analyze('End Scope:');
-
       // run gesture analysis last
       let gestureRes: any[] = [];
       if (this.config.gesture.enabled) {

src/nanodet/nanodet.ts

@@ -1,6 +1,5 @@
 import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as profile from '../profile';
 import { labels } from './labels';
 
 let model;
@@ -83,7 +82,7 @@ async function process(res, inputSize, outputShape, config) {
   const nmsScores = results.map((a) => a.score);
   let nmsIdx: any[] = [];
   if (nmsBoxes && nmsBoxes.length > 0) {
-    const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxResults, config.object.iouThreshold, config.object.minConfidence);
+    const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
     nmsIdx = nms.dataSync();
     tf.dispose(nms);
   }
@@ -114,13 +113,7 @@ export async function predict(image, config) {
     resize.dispose();
 
     let objectT;
-    if (!config.profile) {
     if (config.object.enabled) objectT = await model.predict(transpose);
-    } else {
-      const profileObject = config.object.enabled ? await tf.profile(() => model.predict(transpose)) : {};
-      objectT = profileObject.result;
-      profile.run('object', profileObject);
-    }
     transpose.dispose();
 
     const obj = await process(objectT, model.inputSize, outputSize, config);

src/posenet/poses.ts

@@ -3,6 +3,7 @@ import * as kpt from './keypoints';
 
 const localMaximumRadius = 1;
 const defaultOutputStride = 16;
+const squaredNmsRadius = 20 ** 2;
 
 function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
   const getDisplacement = (point) => ({
@@ -86,7 +87,7 @@ function scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scor
   return localMaximum;
 }
 
-export function buildPartWithScoreQueue(scoreThreshold, scores) {
+export function buildPartWithScoreQueue(minConfidence, scores) {
   const [height, width, numKeypoints] = scores.shape;
   const queue = new utils.MaxHeap(height * width * numKeypoints, ({ score }) => score);
   for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
@@ -94,7 +95,7 @@ export function buildPartWithScoreQueue(scoreThreshold, scores) {
       for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
         const score = scores.get(heatmapY, heatmapX, keypointId);
         // Only consider parts with score greater or equal to threshold as root candidates.
-        if (score < scoreThreshold) continue;
+        if (score < minConfidence) continue;
         // Only consider keypoints whose score is maximum in a local window.
         if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scores)) queue.enqueue({ score, part: { heatmapY, heatmapX, id: keypointId } });
       }
@@ -103,38 +104,37 @@ export function buildPartWithScoreQueue(scoreThreshold, scores) {
   return queue;
 }
 
-function withinRadius(poses, squaredNmsRadius, { x, y }, keypointId) {
+function withinRadius(poses, { x, y }, keypointId) {
   return poses.some(({ keypoints }) => {
     const correspondingKeypoint = keypoints[keypointId].position;
     return utils.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= squaredNmsRadius;
   });
 }
 
-function getInstanceScore(existingPoses, squaredNmsRadius, instanceKeypoints) {
+function getInstanceScore(existingPoses, instanceKeypoints) {
   const notOverlappedKeypointScores = instanceKeypoints.reduce((result, { position, score }, keypointId) => {
-    if (!withinRadius(existingPoses, squaredNmsRadius, position, keypointId)) result += score;
+    if (!withinRadius(existingPoses, position, keypointId)) result += score;
     return result;
   }, 0.0);
   return notOverlappedKeypointScores / instanceKeypoints.length;
 }
 
-export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, nmsRadius, maxDetections, scoreThreshold) {
+export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, maxDetected, minConfidence) {
   const poses: Array<{ keypoints: any, box: any, score: number }> = [];
-  const queue = buildPartWithScoreQueue(scoreThreshold, scoresBuffer);
+  const queue = buildPartWithScoreQueue(minConfidence, scoresBuffer);
-  const squaredNmsRadius = nmsRadius ** 2;
+  // Generate at most maxDetected object instances per image in decreasing root part score order.
-  // Generate at most maxDetections object instances per image in decreasing root part score order.
+  while (poses.length < maxDetected && !queue.empty()) {
-  while (poses.length < maxDetections && !queue.empty()) {
     // The top element in the queue is the next root candidate.
     const root = queue.dequeue();
     // Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
     const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
-    if (withinRadius(poses, squaredNmsRadius, rootImageCoords, root.part.id)) continue;
+    if (withinRadius(poses, rootImageCoords, root.part.id)) continue;
     // Else start a new detection instance at the position of the root.
     const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
-    const keypoints = allKeypoints.filter((a) => a.score > scoreThreshold);
+    const keypoints = allKeypoints.filter((a) => a.score > minConfidence);
-    const score = getInstanceScore(poses, squaredNmsRadius, keypoints);
+    const score = getInstanceScore(poses, keypoints);
     const box = utils.getBoundingBox(keypoints);
-    if (score > scoreThreshold) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });
+    if (score > minConfidence) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });
   }
   return poses;
 }

src/profile.ts

@@ -4,7 +4,7 @@ export const data = {};
 
 export function run(modelName: string, profileData: any): void {
   if (!profileData || !profileData.kernels) return;
-  const maxResults = 5;
+  const maxDetected = 5;
   const time = profileData.kernels
     .filter((a) => a.kernelTimeMs > 0)
     .reduce((a, b) => a += b.kernelTimeMs, 0);
@@ -16,8 +16,8 @@ export function run(modelName: string, profileData: any): void {
     .map((a, i) => { a.id = i; return a; })
     .filter((a) => a.totalBytesSnapshot > 0)
     .sort((a, b) => b.totalBytesSnapshot - a.totalBytesSnapshot);
-  if (slowest.length > maxResults) slowest.length = maxResults;
+  if (slowest.length > maxDetected) slowest.length = maxDetected;
-  if (largest.length > maxResults) largest.length = maxResults;
+  if (largest.length > maxDetected) largest.length = maxDetected;
   data[modelName] = {
     model: modelName,
     newBytes: profileData.newBytes,

wiki

@@ -1 +1 @@
-Subproject commit 3b81af15f2560de5c06f20cbd8de57caf62682f2
+Subproject commit 906244487754b61fd24f49fe2db91ea68264137d