full rebuild

Signed-off-by: Vladimir Mandic <mandic00@live.com>

.api-extractor.json

@@ -1,9 +1,8 @@
 {
   "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
   "mainEntryPointFilePath": "types/lib/src/human.d.ts",
-  "bundledPackages": ["@types/offscreencanvas", "@tensorflow/tfjs-core", "@tensorflow/tfjs-converter", "@tensorflow/tfjs-data"],
   "compiler": {
-    "skipLibCheck": false
+    "skipLibCheck": true
   },
   "newlineKind": "lf",
   "dtsRollup": {

.build.json

@@ -12,10 +12,10 @@
     "clean": ["clean"]
   },
   "clean": {
-    "locations": ["dist/*", "types/lib/*", "typedoc/*"]
+    "locations": ["dist/*", "types/*", "typedoc/*"]
   },
   "lint": {
-    "locations": [ "*.json", "src/**/*.ts", "test/**/*.js", "demo/**/*.js" ],
+    "locations": [ "**/*.json", "src/**/*.ts", "test/**/*.js", "demo/**/*.js", "**/*.md" ],
     "rules": { }
   },
   "changelog": {
@@ -24,8 +24,8 @@
   "serve": {
     "sslKey": "node_modules/@vladmandic/build/cert/https.key",
     "sslCrt": "node_modules/@vladmandic/build/cert/https.crt",
-    "httpPort": 10030,
-    "httpsPort": 10031,
+    "httpPort": 8000,
+    "httpsPort": 8001,
     "documentRoot": ".",
     "defaultFolder": "demo",
     "defaultFile": "index.html"
@@ -39,6 +39,13 @@
       "banner": { "js": "/*\n  Human\n  homepage: <https://github.com/vladmandic/human>\n  author: <https://github.com/vladmandic>'\n*/\n" }
     },
     "targets": [
+      { 
+        "name": "tfjs/browser/version",
+        "platform": "browser",
+        "format": "esm",
+        "input": "tfjs/tf-version.ts",
+        "output": "dist/tfjs.version.js"
+      },
       {
         "name": "tfjs/nodejs/cpu",
         "platform": "node",
@@ -77,6 +84,7 @@
         "format": "cjs",
         "input": "tfjs/tf-node-wasm.ts",
         "output": "dist/tfjs.esm.js",
+        "minify": false,
         "external": ["@tensorflow"]
       },
       {
@@ -87,13 +95,6 @@
         "output": "dist/human.node-wasm.js",
         "external": ["@tensorflow"]
       },
-      { 
-        "name": "tfjs/browser/version",
-        "platform": "browser",
-        "format": "esm",
-        "input": "tfjs/tf-version.ts",
-        "output": "dist/tfjs.version.js"
-      },
       {
         "name": "tfjs/browser/esm/nobundle",
         "platform": "browser",
@@ -108,17 +109,17 @@
         "format": "esm",
         "input": "src/human.ts",
         "output": "dist/human.esm-nobundle.js",
-        "sourcemap": true,
+        "sourcemap": false,
         "external": ["@tensorflow"]
       },
       {
-        "name": "tfjs/browser/esm/custom",
+        "name": "tfjs/browser/esm/bundle",
         "platform": "browser",
         "format": "esm",
-        "input": "tfjs/tf-custom.ts",
+        "input": "tfjs/tf-browser.ts",
         "output": "dist/tfjs.esm.js",
         "sourcemap": false,
-        "minify": false
+        "minify": true
       },
       {
         "name": "human/browser/iife/bundle",
@@ -159,6 +160,15 @@
         "output": "demo/faceid/index.js",
         "sourcemap": true,
         "external": ["*/human.esm.js"]
+      },
+      {
+        "name": "demo/tracker",
+        "platform": "browser",
+        "format": "esm",
+        "input": "demo/tracker/index.ts",
+        "output": "demo/tracker/index.js",
+        "sourcemap": true,
+        "external": ["*/human.esm.js"]
       }
     ]
   },

.eslintrc.json

@@ -1,29 +1,28 @@
 {
-  "globals": {},
+  "globals": {
+  },
+  "rules": {
+    "@typescript-eslint/no-require-imports":"off"
+  },
   "overrides": [
     {
       "files": ["**/*.ts"],
+      "parser": "@typescript-eslint/parser",
+      "parserOptions": { "ecmaVersion": "latest", "project": ["./tsconfig.json"] },
+      "plugins": ["@typescript-eslint"],
       "env": {
         "browser": true,
         "commonjs": false,
         "node": false,
         "es2021": true
       },
-      "parser": "@typescript-eslint/parser",
-      "parserOptions": {
-        "ecmaVersion": "latest",
-        "project": ["./tsconfig.json"]
-      },
-      "plugins": [
-        "@typescript-eslint"
-      ],
       "extends": [
         "airbnb-base",
         "eslint:recommended",
-        "plugin:@typescript-eslint/eslint-recommended",
+        "plugin:@typescript-eslint/strict",
         "plugin:@typescript-eslint/recommended",
         "plugin:@typescript-eslint/recommended-requiring-type-checking",
-        "plugin:@typescript-eslint/strict",
+        "plugin:@typescript-eslint/eslint-recommended",
         "plugin:import/recommended",
         "plugin:promise/recommended"
       ],
@@ -31,9 +30,12 @@
         "@typescript-eslint/ban-ts-comment":"off",
         "@typescript-eslint/dot-notation":"off",
         "@typescript-eslint/no-empty-interface":"off",
+        "@typescript-eslint/no-empty-object-type":"off",
         "@typescript-eslint/no-inferrable-types":"off",
         "@typescript-eslint/no-misused-promises":"off",
+        "@typescript-eslint/no-require-imports":"off",
         "@typescript-eslint/no-unnecessary-condition":"off",
+        "@typescript-eslint/no-unnecessary-type-assertion":"off",
         "@typescript-eslint/no-unsafe-argument":"off",
         "@typescript-eslint/no-unsafe-assignment":"off",
         "@typescript-eslint/no-unsafe-call":"off",
@@ -70,20 +72,59 @@
         "radix":"off"
        }
     },
+    {
+      "files": ["**/*.d.ts"],
+      "parser": "@typescript-eslint/parser",
+      "parserOptions": { "ecmaVersion": "latest", "project": ["./tsconfig.json"] },
+      "plugins": ["@typescript-eslint"],
+      "env": {
+        "browser": true,
+        "commonjs": false,
+        "node": false,
+        "es2021": true
+      },
+      "extends": [
+        "airbnb-base",
+        "eslint:recommended",
+        "plugin:@typescript-eslint/eslint-recommended",
+        "plugin:@typescript-eslint/recommended",
+        "plugin:@typescript-eslint/recommended-requiring-type-checking",
+        "plugin:@typescript-eslint/strict",
+        "plugin:import/recommended",
+        "plugin:promise/recommended"
+      ],
+      "rules": {
+        "@typescript-eslint/array-type":"off",
+        "@typescript-eslint/ban-types":"off",
+        "@typescript-eslint/consistent-indexed-object-style":"off",
+        "@typescript-eslint/consistent-type-definitions":"off",
+        "@typescript-eslint/no-empty-interface":"off",
+        "@typescript-eslint/no-explicit-any":"off",
+        "@typescript-eslint/no-invalid-void-type":"off",
+        "@typescript-eslint/no-unnecessary-type-arguments":"off",
+        "@typescript-eslint/no-unnecessary-type-constraint":"off",
+        "comma-dangle":"off",
+        "indent":"off",
+        "lines-between-class-members":"off",
+        "max-classes-per-file":"off",
+        "max-len":"off",
+        "no-multiple-empty-lines":"off",
+        "no-shadow":"off",
+        "no-use-before-define":"off",
+        "quotes":"off",
+        "semi":"off"
+       }
+    },
     {
       "files": ["**/*.js"],
+      "parserOptions": { "sourceType": "module", "ecmaVersion": "latest" },
+      "plugins": [],
       "env": {
         "browser": true,
         "commonjs": true,
         "node": true,
         "es2021": true
       },
-      "parserOptions": {
-        "sourceType": "module",
-        "ecmaVersion": "latest"
-      },
-      "plugins": [
-      ],
       "extends": [
         "airbnb-base",
         "eslint:recommended",
@@ -113,42 +154,40 @@
     },
     {
       "files": ["**/*.json"],
+      "parserOptions": { "ecmaVersion": "latest" },
+      "plugins": ["json"],
       "env": {
         "browser": false,
         "commonjs": false,
         "node": false,
         "es2021": false
       },
-      "parserOptions": {
-        "ecmaVersion": "latest"
-      },
-      "plugins": [
-        "json"
-      ],
-      "extends": [
-        "plugin:json/recommended"
-      ] 
+      "extends": [] 
     },
     {
-      "files": ["**/*.html"],
-      "env": {
-        "browser": true,
-        "commonjs": false,
-        "node": false,
-        "es2021": false
-      },
-      "parserOptions": {
-        "sourceType": "module",
-        "ecmaVersion": "latest"
-      },
-      "parser": "@html-eslint/parser",
-      "extends": ["plugin:@html-eslint/recommended"],
-      "plugins": [
-        "html", "@html-eslint"
-      ],
+      "files": ["**/*.md"],
+      "plugins": ["markdown"],
+      "processor": "markdown/markdown",
       "rules": {
-        "@html-eslint/element-newline":"off",
-        "@html-eslint/indent": ["error", 2]
+        "no-undef":"off"
+      }
+    },
+    {
+      "files": ["**/*.md/*.js"],
+      "rules": {
+        "@typescript-eslint/no-unused-vars":"off",
+        "@typescript-eslint/triple-slash-reference":"off",
+        "import/newline-after-import":"off",
+        "import/no-unresolved":"off",
+        "no-console":"off",
+        "no-global-assign":"off",
+        "no-multi-spaces":"off",
+        "no-restricted-globals":"off",
+        "no-undef":"off",
+        "no-unused-vars":"off",
+        "node/no-missing-import":"off",
+        "node/no-missing-require":"off",
+        "promise/catch-or-return":"off"
       }
     }
   ],
@@ -159,6 +198,7 @@
     "demo/helpers/*.js",
     "demo/typescript/*.js",
     "demo/faceid/*.js",
+    "demo/tracker/*.js",
     "typedoc"
   ]
 }

.github/FUNDING.yml

@@ -0,0 +1,11 @@
+github: [vladmandic]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.gitignore

@@ -1,4 +1,9 @@
 node_modules/
 types/lib
 pnpm-lock.yaml
+package-lock.json
 *.swp
+samples/**/*.mp4
+samples/**/*.webm
+temp
+tmp

.markdownlint.json

@@ -1,6 +1,7 @@
 {
   "MD012": false,
   "MD013": false,
+  "MD029": false,
   "MD033": false,
   "MD036": false,
   "MD041": false

.npmrc

@@ -1,4 +1,5 @@
 force=true
-production=true
+omit=dev
 legacy-peer-deps=true
 strict-peer-dependencies=false
+node-options='--no-deprecation'

.vscode/settings.json

@@ -1,7 +1,9 @@
 {
   "search.exclude": {
     "dist/*": true,
-    "node_modules/*": true
+    "node_modules/*": true,
+    "types": true,
+    "typedoc": true,
   },
   "search.useGlobalIgnoreFiles": true,
   "search.useParentIgnoreFiles": true

CHANGELOG.md

@@ -1,6 +1,6 @@
 # @vladmandic/human  
 
-  Version: **2.10.1**  
+  Version: **3.3.6**  
   Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**  
   
   Author: **Vladimir Mandic <mandic00@live.com>**  
@@ -9,11 +9,148 @@
   
 ## Changelog
   
-### **HEAD -> main** 2022/09/07 mandic00@live.com
+### **3.3.6** 2025/08/26 mandic00@live.com
 
 
-### **origin/main** 2022/09/07 mandic00@live.com
+### **origin/main** 2025/02/05 mandic00@live.com
 
+- full rebuild
+
+### **3.3.5** 2025/02/05 mandic00@live.com
+
+- rebuild
+- add human.draw.tensor method
+
+### **3.3.4** 2024/10/24 mandic00@live.com
+
+
+### **3.3.3** 2024/10/14 mandic00@live.com
+
+- add loaded property to model stats and mark models not loaded correctly.
+- release build
+
+### **3.3.2** 2024/09/11 mandic00@live.com
+
+- full rebuild
+
+### **3.3.1** 2024/09/11 mandic00@live.com
+
+- add config.face.detector.square option
+- human 3.3 alpha test run
+- human 3.3 alpha with new build environment
+- release rebuild
+- fix flazeface tensor scale and update build platform
+
+### **3.2.2** 2024/04/17 mandic00@live.com
+
+
+### **release: 3.2.1** 2024/02/15 mandic00@live.com
+
+
+### **3.2.1** 2024/02/15 mandic00@live.com
+
+
+### **3.2.0** 2023/12/06 mandic00@live.com
+
+- set browser false when navigator object is empty
+- https://github.com/vladmandic/human/issues/402
+
+### **release: 3.1.2** 2023/09/18 mandic00@live.com
+
+- full rebuild
+
+### **3.1.2** 2023/09/18 mandic00@live.com
+
+- major toolkit upgrade
+- full rebuild
+- major toolkit upgrade
+
+### **3.1.1** 2023/08/05 mandic00@live.com
+
+- fixes plus tfjs upgrade for new release
+
+### **3.0.7** 2023/06/12 mandic00@live.com
+
+- full rebuild
+- fix memory leak in histogramequalization
+- initial work on tracker
+
+### **3.0.6** 2023/03/21 mandic00@live.com
+
+- add optional crop to multiple models
+- fix movenet-multipose
+- add electron detection
+- fix gender-ssrnet-imdb
+- add movenet-multipose workaround
+- rebuild and publish
+- add face.detector.minsize configurable setting
+- add affectnet
+
+### **3.0.5** 2023/02/02 mandic00@live.com
+
+- add gear-e models
+- detect react-native
+- redo blazeface annotations
+
+### **3.0.4** 2023/01/29 mandic00@live.com
+
+- make naviator calls safe
+- fix facedetector-only configs
+
+### **3.0.3** 2023/01/07 mandic00@live.com
+
+- full rebuild
+
+### **3.0.2** 2023/01/06 mandic00@live.com
+
+- default face.rotation disabled
+
+### **release: 3.0.1** 2022/11/22 mandic00@live.com
+
+
+### **3.0.1** 2022/11/22 mandic00@live.com
+
+- support dynamic loads
+- polish demos
+- add facedetect demo and fix model async load
+- enforce markdown linting
+- cleanup git history
+- default empty result
+- refactor draw and models namespaces
+- refactor distance
+- add basic anthropometry
+- added webcam id specification
+- include external typedefs
+- prepare external typedefs
+- rebuild all
+- include project files for types
+- architectural improvements
+- refresh dependencies
+- add named exports
+- add draw label templates
+- reduce dev dependencies
+- tensor rank strong typechecks
+- rebuild dependencies
+
+### **2.11.1** 2022/10/09 mandic00@live.com
+
+- add rvm segmentation model
+- add human.webcam methods
+- create funding.yml
+- fix rotation interpolation
+
+### **2.10.3** 2022/09/21 mandic00@live.com
+
+- add human.video method
+
+### **2.10.2** 2022/09/11 mandic00@live.com
+
+- add node.js esm compatibility (#292)
+- release
+
+### **2.10.1** 2022/09/07 mandic00@live.com
+
+- release candidate
 - add config flags
 - test update
 - release preview
@@ -45,9 +182,7 @@
 
 ### **2.9.2** 2022/08/08 mandic00@live.com
 
-
-### **release: 2.9.1** 2022/07/25 mandic00@live.com
-
+- release rebuild
 
 ### **2.9.1** 2022/07/25 mandic00@live.com
 
@@ -94,7 +229,6 @@
 - enable precompile as part of warmup
 - prepare release beta
 - change default face crop
-- face attention model is available in human-models
 - beta release 2.7
 - refactor draw methods
 - implement face attention model
@@ -406,7 +540,6 @@
 - implemented human.next global interpolation method
 - finished draw buffering and smoothing and enabled by default
 - implemented service worker
-- quantized centernet
 - release candidate
 - added usage restrictions
 - quantize handdetect model

README.md

@@ -1,9 +1,9 @@
+[![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/vladmandic)
 ![Git Version](https://img.shields.io/github/package-json/v/vladmandic/human?style=flat-square&svg=true&label=git)
 ![NPM Version](https://img.shields.io/npm/v/@vladmandic/human.png?style=flat-square)
 ![Last Commit](https://img.shields.io/github/last-commit/vladmandic/human?style=flat-square&svg=true)
 ![License](https://img.shields.io/github/license/vladmandic/human?style=flat-square&svg=true)
 ![GitHub Status Checks](https://img.shields.io/github/checks-status/vladmandic/human/main?style=flat-square&svg=true)
-![Vulnerabilities](https://img.shields.io/snyk/vulnerabilities/github/vladmandic/human?style=flat-square&svg=true)
 
 # Human Library
 
@@ -13,19 +13,45 @@
 
 <br>
 
-JavaScript module using TensorFlow/JS Machine Learning library  
+## Highlights
 
-- **Browser**:  
-  Compatible with both desktop and mobile platforms  
-  Compatible with *CPU*, *WebGL*, *WASM* backends  
-  Compatible with *WebWorker* execution  
-  Compatible with *WebView*  
-- **NodeJS**:  
-  Compatible with both software *tfjs-node* and  
-  GPU accelerated backends *tfjs-node-gpu* using CUDA libraries  
+- Compatible with most server-side and client-side environments and frameworks  
+- Combines multiple machine learning models which can be switched on-demand depending on the use-case  
+- Related models are executed in an attention pipeline to provide details when needed  
+- Optimized input pre-processing that can enhance image quality of any type of inputs  
+- Detection of frame changes to trigger only required models for improved performance  
+- Intelligent temporal interpolation to provide smooth results regardless of processing performance  
+- Simple unified API  
+- Built-in Image, Video and WebCam handling
+
+[*Jump to Quick Start*](#quick-start)
 
 <br>
 
+## Compatibility
+
+**Browser**:  
+  - Compatible with both desktop and mobile platforms  
+  - Compatible with *WebGPU*, *WebGL*, *WASM*, *CPU* backends  
+  - Compatible with *WebWorker* execution  
+  - Compatible with *WebView*  
+  - Primary platform: *Chromium*-based browsers  
+  - Secondary platform: *Firefox*, *Safari*
+
+**NodeJS**:  
+  - Compatibile with *WASM* backend for executions on architectures where *tensorflow* binaries are not available  
+  - Compatible with *tfjs-node* using software execution via *tensorflow* shared libraries  
+  - Compatible with *tfjs-node* using GPU-accelerated execution via *tensorflow* shared libraries and nVidia CUDA  
+  - Supported versions are from **14.x** to **22.x**  
+  - NodeJS version **23.x** is not supported due to breaking changes and issues with `@tensorflow/tfjs`  
+
+<br>
+
+## Releases
+- [Release Notes](https://github.com/vladmandic/human/releases)
+- [NPM Link](https://www.npmjs.com/package/@vladmandic/human)
+## Demos
+
 *Check out [**Simple Live Demo**](https://vladmandic.github.io/human/demo/typescript/index.html) fully annotated app as a good start starting point ([html](https://github.com/vladmandic/human/blob/main/demo/typescript/index.html))([code](https://github.com/vladmandic/human/blob/main/demo/typescript/index.ts))*  
 
 *Check out [**Main Live Demo**](https://vladmandic.github.io/human/demo/index.html) app for advanced processing of of webcam, video stream or images static images with all possible tunable options*  
@@ -33,14 +59,10 @@ JavaScript module using TensorFlow/JS Machine Learning library
 - To start video detection, simply press *Play*  
 - To process images, simply drag & drop in your Browser window  
 - Note: For optimal performance, select only models you'd like to use
-- Note: If you have modern GPU, WebGL (default) backend is preferred, otherwise select WASM backend
+- Note: If you have modern GPU, *WebGL* (default) backend is preferred, otherwise select *WASM* backend
 
 <br>
 
-## Releases
-- [Release Notes](https://github.com/vladmandic/human/releases)
-- [NPM Link](https://www.npmjs.com/package/@vladmandic/human)
-## Demos
 
 - [**List of all Demo applications**](https://github.com/vladmandic/human/wiki/Demos)
 - [**Live Examples galery**](https://vladmandic.github.io/human/samples/index.html)
@@ -51,35 +73,39 @@ JavaScript module using TensorFlow/JS Machine Learning library
 
 - **Full** [[*Live*]](https://vladmandic.github.io/human/demo/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo): Main browser demo app that showcases all Human capabilities
 - **Simple** [[*Live*]](https://vladmandic.github.io/human/demo/typescript/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/typescript): Simple demo in WebCam processing demo in TypeScript
-- **Face Match** [[*Live*]](https://vladmandic.github.io/human/demo/facematch/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/facematch): Extract faces from images, calculates face descriptors and simmilarities and matches them to known database
+- **Embedded** [[*Live*]](https://vladmandic.github.io/human/demo/video/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/video/index.html): Even simpler demo with tiny code embedded in HTML file
+- **Face Detect** [[*Live*]](https://vladmandic.github.io/human/demo/facedetect/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/facedetect): Extract faces from images and processes details
+- **Face Match** [[*Live*]](https://vladmandic.github.io/human/demo/facematch/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/facematch): Extract faces from images, calculates face descriptors and similarities and matches them to known database
 - **Face ID** [[*Live*]](https://vladmandic.github.io/human/demo/faceid/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/faceid): Runs multiple checks to validate webcam input before performing face match to faces in IndexDB
 - **Multi-thread** [[*Live*]](https://vladmandic.github.io/human/demo/multithread/index.html) [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/multithread): Runs each Human module in a separate web worker for highest possible performance  
 - **NextJS** [[*Live*]](https://vladmandic.github.io/human-next/out/index.html) [[*Details*]](https://github.com/vladmandic/human-next): Use Human with TypeScript, NextJS and ReactJS
 - **ElectronJS** [[*Details*]](https://github.com/vladmandic/human-electron): Use Human with TypeScript and ElectonJS to create standalone cross-platform apps
-- **3D Analysis** [[*Live*]](https://vladmandic.github.io/human-motion/src/index.html) [[*Details*]](https://github.com/vladmandic/human-motion): 3D tracking and visualization of heead, face, eye, body and hand
-- **Avatar Bone Mapping** [[*Live*]](https://vladmandic.github.io/human-vrm/src/human-avatar.html) [[*Details*]](https://github.com/vladmandic/human-avatar): Human skeleton with full bone mapping using look and inverse kinematics controllers
-- **Virtual Model Tracking** [[*Live*]](https://vladmandic.github.io/human-vrm/src/human-vrm.html) [[*Details*]](https://github.com/vladmandic/human-vrm): VR model with head, face, eye, body and hand tracking  
+- **3D Analysis with BabylonJS** [[*Live*]](https://vladmandic.github.io/human-motion/src/index.html) [[*Details*]](https://github.com/vladmandic/human-motion): 3D tracking and visualization of heead, face, eye, body and hand
+- **VRM Virtual Model Tracking with Three.JS** [[*Live*]](https://vladmandic.github.io/human-three-vrm/src/human-vrm.html) [[*Details*]](https://github.com/vladmandic/human-three-vrm): VR model with head, face, eye, body and hand tracking  
+- **VRM Virtual Model Tracking with BabylonJS** [[*Live*]](https://vladmandic.github.io/human-bjs-vrm/src/index.html) [[*Details*]](https://github.com/vladmandic/human-bjs-vrm): VR model with head, face, eye, body and hand tracking  
 
 ### NodeJS Demos
 
 *NodeJS demos may require extra dependencies which are used to decode inputs*  
 *See header of each demo to see its dependencies as they are not automatically installed with `Human`*
 
-- **Main** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs): Process images from files, folders or URLs using native methods  
-- **Canvas** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs): Process image from file or URL and draw results to a new image file using `node-canvas`  
-- **Video** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs): Processing of video input using `ffmpeg`  
-- **WebCam** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs): Processing of webcam screenshots using `fswebcam`  
-- **Events** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs): Showcases usage of `Human` eventing to get notifications on processing
-- **Similarity** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs): Compares two input images for similarity of detected faces
-- **Face Match** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/facematch): Parallel processing of face **match** in multiple child worker threads
-- **Multiple Workers** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs): Runs multiple parallel `human` by dispaching them to pool of pre-created worker processes  
+- **Main** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs/node.js): Process images from files, folders or URLs using native methods  
+- **Canvas** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs/node-canvas.js): Process image from file or URL and draw results to a new image file using `node-canvas`  
+- **Video** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs/node-video.js): Processing of video input using `ffmpeg`  
+- **WebCam** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs/node-webcam.js): Processing of webcam screenshots using `fswebcam`  
+- **Events** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs/node-event.js): Showcases usage of `Human` eventing to get notifications on processing
+- **Similarity** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs/node-similarity.js): Compares two input images for similarity of detected faces
+- **Face Match** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/facematch/node-match.js): Parallel processing of face **match** in multiple child worker threads
+- **Multiple Workers** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/multithread/node-multiprocess.js): Runs multiple parallel `human` by dispaching them to pool of pre-created worker processes  
+- **Dynamic Load** [[*Details*]](https://github.com/vladmandic/human/tree/main/demo/nodejs): Loads Human dynamically with multiple different desired backends  
 
 ## Project pages
 
 - [**Code Repository**](https://github.com/vladmandic/human)
 - [**NPM Package**](https://www.npmjs.com/package/@vladmandic/human)
 - [**Issues Tracker**](https://github.com/vladmandic/human/issues)
-- [**TypeDoc API Specification**](https://vladmandic.github.io/human/typedoc/classes/Human.html)
+- [**TypeDoc API Specification - Main class**](https://vladmandic.github.io/human/typedoc/classes/Human.html)
+- [**TypeDoc API Specification - Full**](https://vladmandic.github.io/human/typedoc/)
 - [**Change Log**](https://github.com/vladmandic/human/blob/main/CHANGELOG.md)
 - [**Current To-do List**](https://github.com/vladmandic/human/blob/main/TODO.md)
 
@@ -90,6 +116,7 @@ JavaScript module using TensorFlow/JS Machine Learning library
 - [**Usage & Functions**](https://github.com/vladmandic/human/wiki/Usage)
 - [**Configuration Details**](https://github.com/vladmandic/human/wiki/Config)
 - [**Result Details**](https://github.com/vladmandic/human/wiki/Result)
+- [**Customizing Draw Methods**](https://github.com/vladmandic/human/wiki/Draw)
 - [**Caching & Smoothing**](https://github.com/vladmandic/human/wiki/Caching)
 - [**Input Processing**](https://github.com/vladmandic/human/wiki/Image)
 - [**Face Recognition & Face Description**](https://github.com/vladmandic/human/wiki/Embedding)
@@ -121,27 +148,24 @@ JavaScript module using TensorFlow/JS Machine Learning library
 
 <hr><br>
 
-## Examples
+## App Examples
 
-Visit [Examples galery](https://vladmandic.github.io/human/samples/samples.html) for more examples  
-<https://vladmandic.github.io/human/samples/samples.html>
-
-![samples](assets/samples.jpg)
+Visit [Examples gallery](https://vladmandic.github.io/human/samples/index.html) for more examples  
+[<img src="assets/samples.jpg" width="640"/>](assets/samples.jpg)
 
 <br>
 
 ## Options
 
 All options as presented in the demo application...  
-> [demo/index.html](demo/index.html)  
-
-![Options visible in demo](assets/screenshot-menu.png)
+[demo/index.html](demo/index.html)  
+[<img src="assets/screenshot-menu.png"/>](assets/screenshot-menu.png)
 
 <br>
 
 **Results Browser:**  
 [ *Demo -> Display -> Show Results* ]<br>
-![Results](assets/screenshot-results.png)
+[<img src="assets/screenshot-results.png"/>](assets/screenshot-results.png)
 
 <br>
 
@@ -153,37 +177,47 @@ sorts them by similarity to selected face
 and optionally matches detected face with database of known people to guess their names
 > [demo/facematch](demo/facematch/index.html)  
 
-![Face Matching](assets/screenshot-facematch.jpg)
+[<img src="assets/screenshot-facematch.jpg" width="640"/>](assets/screenshot-facematch.jpg)
+
+2. **Face Detect:**  
+Extracts all detect faces from loaded images on-demand and highlights face details on a selected face  
+> [demo/facedetect](demo/facedetect/index.html)  
+
+[<img src="assets/screenshot-facedetect.jpg" width="640"/>](assets/screenshot-facedetect.jpg)
+
+3. **Face ID:**  
+Performs validation check on a webcam input to detect a real face and matches it to known faces stored in database
+> [demo/faceid](demo/faceid/index.html)  
+
+[<img src="assets/screenshot-faceid.jpg" width="640"/>](assets/screenshot-faceid.jpg)
 
 <br>
 
-2. **3D Rendering:**  
+4. **3D Rendering:**  
 > [human-motion](https://github.com/vladmandic/human-motion)
 
-![Face3D](https://github.com/vladmandic/human-motion/raw/main/assets/screenshot-face.jpg)
-![Body3D](https://github.com/vladmandic/human-motion/raw/main/assets/screenshot-body.jpg)
-![Hand3D](https://github.com/vladmandic/human-motion/raw/main/assets/screenshot-hand.jpg)
+[<img src="https://github.com/vladmandic/human-motion/raw/main/assets/screenshot-face.jpg" width="640"/>](https://github.com/vladmandic/human-motion/raw/main/assets/screenshot-face.jpg)
+[<img src="https://github.com/vladmandic/human-motion/raw/main/assets/screenshot-body.jpg" width="640"/>](https://github.com/vladmandic/human-motion/raw/main/assets/screenshot-body.jpg)
+[<img src="https://github.com/vladmandic/human-motion/raw/main/assets/screenshot-hand.jpg" width="640"/>](https://github.com/vladmandic/human-motion/raw/main/assets/screenshot-hand.jpg)
 
 <br>
 
-3. **Avatar Bone Mapping:**  
-> [human-avatar](https://github.com/vladmandic/human-avatar)
+5. **VR Model Tracking:**  
+> [human-three-vrm](https://github.com/vladmandic/human-three-vrm)  
+> [human-bjs-vrm](https://github.com/vladmandic/human-bjs-vrm)  
 
-![Avatar](https://github.com/vladmandic/human-avatar/raw/main/assets/screenshot.jpg)
+[<img src="https://github.com/vladmandic/human-three-vrm/raw/main/assets/human-vrm-screenshot.jpg" width="640"/>](https://github.com/vladmandic/human-three-vrm/raw/main/assets/human-vrm-screenshot.jpg)
 
-<br>
 
-4. **VR Model Tracking:**  
-> [human-vrmmotion](https://github.com/vladmandic/human-vrm)
-
-![VRM](https://github.com/vladmandic/human-vrm/raw/main/assets/human-vrm-screenshot.jpg)
+6. **Human as OS native application:**
+> [human-electron](https://github.com/vladmandic/human-electron)
 
 <br>
 
 **468-Point Face Mesh Defails:**  
 (view in full resolution to see keypoints)  
 
-![FaceMesh](assets/facemesh.png)
+[<img src="assets/facemesh.png" width="400"/>](assets/facemesh.png)
 
 <br><hr><br>
 
@@ -193,44 +227,25 @@ Simply load `Human` (*IIFE version*) directly from a cloud CDN in your HTML file
 (pick one: `jsdelirv`, `unpkg` or `cdnjs`)
 
 ```html
+<!DOCTYPE HTML>
 <script src="https://cdn.jsdelivr.net/npm/@vladmandic/human/dist/human.js"></script>
 <script src="https://unpkg.dev/@vladmandic/human/dist/human.js"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/human/2.1.5/human.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/human/3.0.0/human.js"></script>
 ```
 
 For details, including how to use `Browser ESM` version or `NodeJS` version of `Human`, see [**Installation**](https://github.com/vladmandic/human/wiki/Install)
 
 <br>
 
-## Inputs
+## Code Examples
 
-`Human` library can process all known input types:  
-
-- `Image`, `ImageData`, `ImageBitmap`, `Canvas`, `OffscreenCanvas`, `Tensor`,  
-- `HTMLImageElement`, `HTMLCanvasElement`, `HTMLVideoElement`, `HTMLMediaElement`
-
-Additionally, `HTMLVideoElement`, `HTMLMediaElement` can be a standard `<video>` tag that links to:
-
-- WebCam on user's system
-- Any supported video type  
-  For example: `.mp4`, `.avi`, etc.
-- Additional video types supported via *HTML5 Media Source Extensions*  
-  Live streaming examples:
-  - **HLS** (*HTTP Live Streaming*) using `hls.js`
-  - **DASH** (Dynamic Adaptive Streaming over HTTP) using `dash.js`
-- **WebRTC** media track using built-in support  
-
-<br>
-
-## Example
-
-Example simple app that uses Human to process video input and  
+Simple app that uses Human to process video input and  
 draw output on screen using internal draw helper functions
 
 ```js
 // create instance of human with simple configuration using default values
 const config = { backend: 'webgl' };
-const human = new Human(config);
+const human = new Human.Human(config);
 // select input HTMLVideoElement and output HTMLCanvasElement from page
 const inputVideo = document.getElementById('video-id');
 const outputCanvas = document.getElementById('canvas-id');
@@ -249,6 +264,7 @@ function detectVideo() {
     human.draw.gesture(outputCanvas, result.gesture);
     // and loop immediate to the next frame
     requestAnimationFrame(detectVideo);
+    return result;
   });
 }
 
@@ -309,7 +325,7 @@ async function detectVideo() {
 
 async function drawVideo() {
   if (result) { // check if result is available
-    const interpolated = human.next(result); // calculate next interpolated frame
+    const interpolated = human.next(result); // get smoothened result using last-known results
     human.draw.all(outputCanvas, interpolated); // draw the frame
   }
   requestAnimationFrame(drawVideo); // run draw loop
@@ -319,26 +335,108 @@ detectVideo(); // start detection loop
 drawVideo(); // start draw loop
 ```
 
+or same, but using built-in full video processing instead of running manual frame-by-frame loop:
+
+```js
+const human = new Human(); // create instance of Human
+const inputVideo = document.getElementById('video-id');
+const outputCanvas = document.getElementById('canvas-id');
+
+async function drawResults() {
+  const interpolated = human.next(); // get smoothened result using last-known results
+  human.draw.all(outputCanvas, interpolated); // draw the frame
+  requestAnimationFrame(drawResults); // run draw loop
+}
+
+human.video(inputVideo); // start detection loop which continously updates results
+drawResults(); // start draw loop
+```
+
+or using built-in webcam helper methods that take care of video handling completely:
+
+```js
+const human = new Human(); // create instance of Human
+const outputCanvas = document.getElementById('canvas-id');
+
+async function drawResults() {
+  const interpolated = human.next(); // get smoothened result using last-known results
+  human.draw.canvas(outputCanvas, human.webcam.element); // draw current webcam frame
+  human.draw.all(outputCanvas, interpolated); // draw the frame detectgion results
+  requestAnimationFrame(drawResults); // run draw loop
+}
+
+await human.webcam.start({ crop: true });
+human.video(human.webcam.element); // start detection loop which continously updates results
+drawResults(); // start draw loop
+```
+
 And for even better results, you can run detection in a separate web worker thread
 
+<br><hr><br>
+
+## Inputs
+
+`Human` library can process all known input types:  
+
+- `Image`, `ImageData`, `ImageBitmap`, `Canvas`, `OffscreenCanvas`, `Tensor`,  
+- `HTMLImageElement`, `HTMLCanvasElement`, `HTMLVideoElement`, `HTMLMediaElement`
+
+Additionally, `HTMLVideoElement`, `HTMLMediaElement` can be a standard `<video>` tag that links to:
+
+- WebCam on user's system
+- Any supported video type  
+  e.g. `.mp4`, `.avi`, etc.
+- Additional video types supported via *HTML5 Media Source Extensions*  
+  e.g.: **HLS** (*HTTP Live Streaming*) using `hls.js` or **DASH** (*Dynamic Adaptive Streaming over HTTP*) using `dash.js`
+- **WebRTC** media track using built-in support  
+
+<br><hr><br>
+
+## Detailed Usage
+
+- [**Wiki Home**](https://github.com/vladmandic/human/wiki)
+- [**List of all available methods, properies and namespaces**](https://github.com/vladmandic/human/wiki/Usage)
+- [**TypeDoc API Specification - Main class**](https://vladmandic.github.io/human/typedoc/classes/Human.html)
+- [**TypeDoc API Specification - Full**](https://vladmandic.github.io/human/typedoc/)
+
+    ![typedoc](assets/screenshot-typedoc.png)
+
+<br><hr><br>
+
+## TypeDefs
+
+`Human` is written using TypeScript strong typing and ships with full **TypeDefs** for all classes defined by the library bundled in `types/human.d.ts` and enabled by default  
+
+*Note*: This does not include embedded `tfjs`  
+If you want to use embedded `tfjs` inside `Human` (`human.tf` namespace) and still full **typedefs**, add this code:
+
+> import type * as tfjs from '@vladmandic/human/dist/tfjs.esm';  
+> const tf = human.tf as typeof tfjs;
+
+This is not enabled by default as `Human` does not ship with full **TFJS TypeDefs** due to size considerations  
+Enabling `tfjs` TypeDefs as above creates additional project (dev-only as only types are required) dependencies as defined in `@vladmandic/human/dist/tfjs.esm.d.ts`:
+
+> @tensorflow/tfjs-core, @tensorflow/tfjs-converter, @tensorflow/tfjs-backend-wasm, @tensorflow/tfjs-backend-webgl
+
+
 <br><hr><br>
 
 ## Default models
 
 Default models in Human library are:
 
-- **Face Detection**: MediaPipe BlazeFace Back variation
-- **Face Mesh**: MediaPipe FaceMesh
-- **Face Iris Analysis**: MediaPipe Iris
-- **Face Description**: HSE FaceRes
-- **Emotion Detection**: Oarriaga Emotion
-- **Body Analysis**: MoveNet Lightning variation
-- **Hand Analysis**: HandTrack & MediaPipe HandLandmarks
-- **Body Segmentation**: Google Selfie
-- **Object Detection**: CenterNet with MobileNet v3
+- **Face Detection**: *MediaPipe BlazeFace Back variation*
+- **Face Mesh**: *MediaPipe FaceMesh*
+- **Face Iris Analysis**: *MediaPipe Iris*
+- **Face Description**: *HSE FaceRes*
+- **Emotion Detection**: *Oarriaga Emotion*
+- **Body Analysis**: *MoveNet Lightning variation*
+- **Hand Analysis**: *HandTrack & MediaPipe HandLandmarks*
+- **Body Segmentation**: *Google Selfie*
+- **Object Detection**: *CenterNet with MobileNet v3*
 
 Note that alternative models are provided and can be enabled via configuration  
-For example, `PoseNet` model can be switched for `BlazePose`, `EfficientPose` or `MoveNet` model depending on the use case  
+For example, body pose detection by default uses *MoveNet Lightning*, but can be switched to *MultiNet Thunder* for higher precision or *Multinet MultiPose* for multi-person detection or even *PoseNet*, *BlazePose* or *EfficientPose* depending on the use case  
 
 For more info, see [**Configuration Details**](https://github.com/vladmandic/human/wiki/Configuration) and [**List of Models**](https://github.com/vladmandic/human/wiki/Models)
 
@@ -350,9 +448,9 @@ For more info, see [**Configuration Details**](https://github.com/vladmandic/hum
 
 <br><hr><br>
 
-`Human` library is written in `TypeScript` [4.8](https://www.typescriptlang.org/docs/handbook/intro.html)  
-Conforming to latest `JavaScript` [ECMAScript version 2022](https://262.ecma-international.org/) standard  
-Build target is `JavaScript` [EMCAScript version 2018](https://262.ecma-international.org/11.0/)  
+`Human` library is written in [TypeScript](https://www.typescriptlang.org/docs/handbook/intro.html) **5.1** using [TensorFlow/JS](https://www.tensorflow.org/js/) **4.10** and conforming to latest `JavaScript` [ECMAScript version 2022](https://262.ecma-international.org/) standard  
+
+Build target for distributables is `JavaScript` [EMCAScript version 2018](https://262.ecma-international.org/9.0/)  
 
 <br>
 
@@ -361,6 +459,7 @@ and [**API Specification**](https://vladmandic.github.io/human/typedoc/classes/H
 
 <br>
 
+[![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/vladmandic)
 ![Stars](https://img.shields.io/github/stars/vladmandic/human?style=flat-square&svg=true)
 ![Forks](https://badgen.net/github/forks/vladmandic/human)
 ![Code Size](https://img.shields.io/github/languages/code-size/vladmandic/human?style=flat-square&svg=true)

TODO.md

@@ -2,94 +2,37 @@
 
 ## Work-in-Progress
 
-N/A
-
 <hr><br>
 
-## Exploring
-
-- Optical flow for intelligent temporal interpolation  
-  <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
-- Advanced histogram equalization for optimization of badly lit scenes  
-  **Adaptive**, **Contrast Limited**, **CLAHE**
-- TFLite models  
-  <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
-- Body segmentation with temporal analysis  
-  <https://github.com/PeterL1n/RobustVideoMatting>
-
-<hr><br>
-
-## Known Issues
+## Known Issues & Limitations
 
 ### Face with Attention
 
-`FaceMesh-Attention` is not supported in browser using `WASM` backend due to missing kernel op in **TFJS**  
+`FaceMesh-Attention` is not supported when using `WASM` backend due to missing kernel op in **TFJS**  
+No issues with default model `FaceMesh`  
 
 ### Object Detection
 
-`NanoDet` model is not supported in in browser using `WASM` backend due to missing kernel op in **TFJS**  
+`NanoDet` model is not supported when using `WASM` backend due to missing kernel op in **TFJS**  
+No issues with default model `MB3-CenterNet`  
 
-### WebGPU
+## Body Detection using MoveNet-MultiPose
 
-Experimental support only until support is officially added in Chromium  
-Enable via <chrome://flags/#enable-unsafe-webgpu>
+Model does not return valid detection scores (all other functionality is not impacted)  
 
 ### Firefox
 
 Running in **web workers** requires `OffscreenCanvas` which is still disabled by default in **Firefox**  
 Enable via `about:config` -> `gfx.offscreencanvas.enabled`  
+[Details](https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas#browser_compatibility)  
+
+### Safari
+
+No support for running in **web workers** as Safari still does not support `OffscreenCanvas`  
+[Details](https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas#browser_compatibility)  
+
+## React-Native
+
+`Human` support for **React-Native** is best-effort, but not part of the main development focus  
 
 <hr><br>
-
-## Pending Release Changes
-
-- Update **TFJS** to **3.20.0**  
-- Update **TypeScript** to **4.8**  
-- Switch default backend from custom `humangl` to `webgl`  
-  Stability and availability of features in standard **TFJS** allows to switch back  
-- Add **InsightFace** model as alternative for face embedding/descriptor detection  
-  Compatible with multiple variations of **InsightFace** models  
-  Configurable using `config.face.insightface` config section  
-  See `demo/faceid/index.ts` for usage  
-  Models can be downloaded from <https://github.com/vladmandic/insightface>  
-- Add `human.check()` which validates all kernel ops for currently loaded models with currently selected backend  
-  Example: `console.error(human.check());`  
-- Add `config.softwareKernels` config option which uses **CPU** implementation for missing ops  
-  Disabled by default  
-  If enabled, it is used by face and hand rotation correction (`config.face.rotation` and `config.hand.rotation`)  
-- Add underlying **tensorflow** library version detection when running in NodeJS to  
-  `human.env` and check if **GPU** is used for acceleration  
-  Example: `console.log(human.env.tensorflow)`  
-- Treat models that cannot be found & loaded as non-critical error  
-  Instead of creating runtime exception, `human` will now report that model could not be loaded  
-- Improve `human.reset()` method to reset all config values to defaults
-- Host models in <https://github.com/vladmandic/human-models>  
-  Models can be directly used without downloading to local storage  
-  Example: `modelBasePath: 'https://vladmandic.github.io/human-models/models/'`  
-- Allow hosting models in **Google Cloud Bucket**  
-  Hosted models can be directly used without downloading to local storage  
-  Example: `modelBasePath: 'https://storage.googleapis.com/human-models/'`  
-- Stricter linting rules for both **TypeScript** and **JavaScript**  
-  See `./eslintrc.json` for details  
-- Enhanced type safety across entire library  
-- Fix **MobileFaceNet** model as alternative for face embedding/descriptor detection  
-  Configurable using `config.face.mobilefacenet` config section  
-- Fix **EfficientPose** module as alternative body detection  
-- Fix **NanoDet** module as alternative object detection  
-- Fix `demo/multithread/node-multiprocess.js` demo  
-- Fix `human.match` when using mixed descriptor lengths  
-- Fix **WASM** feature detection issue in TFJS with Edge/Chromium  
-  Example: `console.log(human.env.wasm)`  
-- Reorganized init & load order for faster library startup
-- Increased **NodeJS** test coverage  
-  Run using: `npm run test`  
-  Runs tests for `tfjs-node`, `tfjs-node-gpu` and `wasm`  
-- Increased **Browser** test coverage  
-  Run using: `demo/browser.html`  
-  Runs tests for `webgl`, `humangl`, `webgpu` and `wasm`  
-  Runs tests for ESM and IIFE versions of library  
-- Add `config.flags` settings to allow setting of custom **TFJS** flags during backend configuration
-- Increase availability of alternative models  
-  See `models/model.json` for full list  
-- Update profiling methods in `human.profile()`  
-- Update project dependencies to latest versions  

build.js

@@ -31,12 +31,44 @@ const apiExtractorIgnoreList = [ // eslint-disable-line no-unused-vars
   'tsdoc-unnecessary-backslash',
 ];
 
-function copy(src, dst) {
-  if (!fs.existsSync(src)) return;
+const regEx = [
+  { search: 'types="@webgpu/types/dist"', replace: 'path="../src/types/webgpu.d.ts"' },
+  { search: 'types="offscreencanvas"', replace: 'path="../src/types/offscreencanvas.d.ts"' },
+];
+
+function copyFile(src, dst) {
+  if (!fs.existsSync(src)) {
+    log.warn('Copy:', { input: src, output: dst });
+    return;
+  }
+  log.state('Copy:', { input: src, output: dst });
   const buffer = fs.readFileSync(src);
   fs.writeFileSync(dst, buffer);
 }
 
+function writeFile(str, dst) {
+  log.state('Write:', { output: dst });
+  fs.writeFileSync(dst, str);
+}
+
+function regExFile(src, entries) {
+  if (!fs.existsSync(src)) {
+    log.warn('Filter:', { src });
+    return;
+  }
+  log.state('Filter:', { input: src });
+  for (const entry of entries) {
+    const buffer = fs.readFileSync(src, 'UTF-8');
+    const lines = buffer.split(/\r?\n/);
+    const out = [];
+    for (const line of lines) {
+      if (line.includes(entry.search)) out.push(line.replace(entry.search, entry.replace));
+      else out.push(line);
+    }
+    fs.writeFileSync(src, out.join('\n'));
+  }
+}
+
 async function analyzeModels() {
   log.info('Analyze models:', { folders: modelsFolders.length, result: modelsOut });
   let totalSize = 0;
@@ -74,14 +106,22 @@ async function analyzeModels() {
 async function main() {
   log.logFile(logFile);
   log.data('Build', { name: packageJSON.name, version: packageJSON.version });
+
   // run production build
   const build = new Build();
   await build.run('production');
+
   // patch tfjs typedefs
-  log.state('Copy:', { input: 'tfjs/tfjs.esm.d.ts' });
-  copy('tfjs/tfjs.esm.d.ts', 'types/lib/dist/tfjs.esm.d.ts');
+  copyFile('node_modules/@vladmandic/tfjs/types/tfjs-core.d.ts', 'types/tfjs-core.d.ts');
+  copyFile('node_modules/@vladmandic/tfjs/types/tfjs.d.ts', 'types/tfjs.esm.d.ts');
+  copyFile('src/types/tsconfig.json', 'types/tsconfig.json');
+  copyFile('src/types/eslint.json', 'types/.eslintrc.json');
+  copyFile('src/types/tfjs.esm.d.ts', 'dist/tfjs.esm.d.ts');
+  regExFile('types/tfjs-core.d.ts', regEx);
+
   // run api-extractor to create typedef rollup
   const extractorConfig = APIExtractor.ExtractorConfig.loadFileAndPrepare('.api-extractor.json');
+  try {
     const extractorResult = APIExtractor.Extractor.invoke(extractorConfig, {
       localBuild: true,
       showVerboseMessages: false,
@@ -94,14 +134,17 @@ async function main() {
       },
     });
     log.state('API-Extractor:', { succeeeded: extractorResult.succeeded, errors: extractorResult.errorCount, warnings: extractorResult.warningCount });
-  // distribute typedefs
-  log.state('Copy:', { input: 'types/human.d.ts' });
-  copy('types/human.d.ts', 'dist/human.esm-nobundle.d.ts');
-  copy('types/human.d.ts', 'dist/human.esm.d.ts');
-  copy('types/human.d.ts', 'dist/human.d.ts');
-  copy('types/human.d.ts', 'dist/human.node-gpu.d.ts');
-  copy('types/human.d.ts', 'dist/human.node.d.ts');
-  copy('types/human.d.ts', 'dist/human.node-wasm.d.ts');
+  } catch (err) {
+    log.error('API-Extractor:', err);
+  }
+  regExFile('types/human.d.ts', regEx);
+  writeFile('export * from \'../types/human\';', 'dist/human.esm-nobundle.d.ts');
+  writeFile('export * from \'../types/human\';', 'dist/human.esm.d.ts');
+  writeFile('export * from \'../types/human\';', 'dist/human.d.ts');
+  writeFile('export * from \'../types/human\';', 'dist/human.node-gpu.d.ts');
+  writeFile('export * from \'../types/human\';', 'dist/human.node.d.ts');
+  writeFile('export * from \'../types/human\';', 'dist/human.node-wasm.d.ts');
+
   // generate model signature
   await analyzeModels();
   log.info('Human Build complete...', { logFile });

demo/README.md

@@ -8,6 +8,7 @@ For details on other demos see Wiki: [**Demos**](https://github.com/vladmandic/h
 `index.html`: Full demo using `Human` ESM module running in Browsers,  
 
 Includes:
+
 - Selectable inputs:
   - Sample images
   - Image via drag & drop
@@ -37,12 +38,14 @@ Includes:
   in `index.js:ui`
 
 ```js
+const ui = {
   console: true,      // log messages to browser console
   useWorker: true,    // use web workers for processing
   buffered: true,     // should output be buffered between frames
   interpolated: true, // should output be interpolated for smoothness between frames
   results: false,     // show results tree
   useWebRTC: false,   // use webrtc as camera source instead of local webcam
+};
 ```
 
 Demo implements several ways to use `Human` library,  

demo/facedetect/facedetect.js

@@ -0,0 +1,160 @@
+/**
+ * Human demo for browsers
+ *
+ * Demo for face detection
+ */
+
+/** @type {Human} */
+import { Human } from '../../dist/human.esm.js';
+
+let loader;
+
+const humanConfig = { // user configuration for human, used to fine-tune behavior
+  cacheSensitivity: 0,
+  debug: true,
+  modelBasePath: 'https://vladmandic.github.io/human-models/models/',
+  filter: { enabled: true, equalization: false, flip: false },
+  face: {
+    enabled: true,
+    detector: { rotation: false, maxDetected: 100, minConfidence: 0.2, return: true, square: false },
+    iris: { enabled: true },
+    description: { enabled: true },
+    emotion: { enabled: true },
+    antispoof: { enabled: true },
+    liveness: { enabled: true },
+  },
+  body: { enabled: false },
+  hand: { enabled: false },
+  object: { enabled: false },
+  gesture: { enabled: false },
+  segmentation: { enabled: false },
+};
+
+const human = new Human(humanConfig); // new instance of human
+
+export const showLoader = (msg) => { loader.setAttribute('msg', msg); loader.style.display = 'block'; };
+export const hideLoader = () => loader.style.display = 'none';
+
+class ComponentLoader extends HTMLElement { // watch for attributes
+  message = document.createElement('div');
+
+  static get observedAttributes() { return ['msg']; }
+
+  attributeChangedCallback(_name, _prevVal, currVal) {
+    this.message.innerHTML = currVal;
+  }
+
+  connectedCallback() { // triggered on insert
+    this.attachShadow({ mode: 'open' });
+    const css = document.createElement('style');
+    css.innerHTML = `
+      .loader-container { top: 450px; justify-content: center; position: fixed; width: 100%; }
+      .loader-message { font-size: 1.5rem; padding: 1rem; }
+      .loader { width: 300px; height: 300px; border: 3px solid transparent; border-radius: 50%; border-top: 4px solid #f15e41; animation: spin 4s linear infinite; position: relative; }
+      .loader::before, .loader::after { content: ""; position: absolute; top: 6px; bottom: 6px; left: 6px; right: 6px; border-radius: 50%; border: 4px solid transparent; }
+      .loader::before { border-top-color: #bad375; animation: 3s spin linear infinite; }
+      .loader::after { border-top-color: #26a9e0; animation: spin 1.5s linear infinite; }
+      @keyframes spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } }
+    `;
+    const container = document.createElement('div');
+    container.id = 'loader-container';
+    container.className = 'loader-container';
+    loader = document.createElement('div');
+    loader.id = 'loader';
+    loader.className = 'loader';
+    this.message.id = 'loader-message';
+    this.message.className = 'loader-message';
+    this.message.innerHTML = '';
+    container.appendChild(this.message);
+    container.appendChild(loader);
+    this.shadowRoot?.append(css, container);
+    loader = this; // eslint-disable-line @typescript-eslint/no-this-alias
+  }
+}
+
+customElements.define('component-loader', ComponentLoader);
+
+function addFace(face, source) {
+  const deg = (rad) => Math.round((rad || 0) * 180 / Math.PI);
+  const canvas = document.createElement('canvas');
+  const emotion = face.emotion?.map((e) => `${Math.round(100 * e.score)}% ${e.emotion}`) || [];
+  const rotation = `pitch ${deg(face.rotation?.angle.pitch)}° | roll ${deg(face.rotation?.angle.roll)}°  | yaw ${deg(face.rotation?.angle.yaw)}°`;
+  const gaze = `direction ${deg(face.rotation?.gaze.bearing)}° strength ${Math.round(100 * (face.rotation.gaze.strength || 0))}%`;
+  canvas.title = `
+    source: ${source}
+    score: ${Math.round(100 * face.boxScore)}% detection ${Math.round(100 * face.faceScore)}% analysis
+    age: ${face.age} years | gender: ${face.gender} score ${Math.round(100 * face.genderScore)}%
+    emotion: ${emotion.join(' | ')}
+    head rotation: ${rotation}
+    eyes gaze: ${gaze}
+    camera distance: ${face.distance}m | ${Math.round(100 * face.distance / 2.54)}in
+    check: ${Math.round(100 * face.real)}% real ${Math.round(100 * face.live)}% live
+  `.replace(/  /g, ' ');
+  canvas.onclick = (e) => {
+    e.preventDefault();
+    document.getElementById('description').innerHTML = canvas.title;
+  };
+  human.draw.tensor(face.tensor, canvas);
+  human.tf.dispose(face.tensor);
+  return canvas;
+}
+
+async function addFaces(imgEl) {
+  showLoader('human: busy');
+  const faceEl = document.getElementById('faces');
+  faceEl.innerHTML = '';
+  const res = await human.detect(imgEl);
+  console.log(res); // eslint-disable-line no-console
+  document.getElementById('description').innerHTML = `detected ${res.face.length} faces`;
+  for (const face of res.face) {
+    const canvas = addFace(face, imgEl.src.substring(0, 64));
+    faceEl.appendChild(canvas);
+  }
+  hideLoader();
+}
+
+function addImage(imageUri) {
+  const imgEl = new Image(256, 256);
+  imgEl.onload = () => {
+    const images = document.getElementById('images');
+    images.appendChild(imgEl); // add image if loaded ok
+    images.scroll(images?.offsetWidth, 0);
+  };
+  imgEl.onerror = () => console.error('addImage', { imageUri }); // eslint-disable-line no-console
+  imgEl.onclick = () => addFaces(imgEl);
+  imgEl.title = imageUri.substring(0, 64);
+  imgEl.src = encodeURI(imageUri);
+}
+
+async function initDragAndDrop() {
+  const reader = new FileReader();
+  reader.onload = async (e) => {
+    if (e.target.result.startsWith('data:image')) await addImage(e.target.result);
+  };
+  document.body.addEventListener('dragenter', (evt) => evt.preventDefault());
+  document.body.addEventListener('dragleave', (evt) => evt.preventDefault());
+  document.body.addEventListener('dragover', (evt) => evt.preventDefault());
+  document.body.addEventListener('drop', async (evt) => {
+    evt.preventDefault();
+    evt.dataTransfer.dropEffect = 'copy';
+    for (const f of evt.dataTransfer.files) reader.readAsDataURL(f);
+  });
+  document.body.onclick = (e) => {
+    if (e.target.localName !== 'canvas') document.getElementById('description').innerHTML = '';
+  };
+}
+
+async function main() {
+  showLoader('loading models');
+  await human.load();
+  showLoader('compiling models');
+  await human.warmup();
+  showLoader('loading images');
+  const images = ['group-1.jpg', 'group-2.jpg', 'group-3.jpg', 'group-4.jpg', 'group-5.jpg', 'group-6.jpg', 'group-7.jpg', 'solvay1927.jpg', 'stock-group-1.jpg', 'stock-group-2.jpg', 'stock-models-6.jpg', 'stock-models-7.jpg'];
+  const imageUris = images.map((a) => `../../samples/in/${a}`);
+  for (let i = 0; i < imageUris.length; i++) addImage(imageUris[i]);
+  initDragAndDrop();
+  hideLoader();
+}
+
+window.onload = main;

demo/facedetect/index.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Human</title>
+    <!-- <meta http-equiv="content-type" content="text/html; charset=utf-8"> -->
+    <meta name="viewport" content="width=device-width, shrink-to-fit=yes">
+    <meta name="keywords" content="Human">
+    <meta name="application-name" content="Human">
+    <meta name="description" content="Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition; Author: Vladimir Mandic <https://github.com/vladmandic>">
+    <meta name="msapplication-tooltip" content="Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition; Author: Vladimir Mandic <https://github.com/vladmandic>">
+    <meta name="theme-color" content="#000000">
+    <link rel="manifest" href="../manifest.webmanifest">
+    <link rel="shortcut icon" href="../../favicon.ico" type="image/x-icon">
+    <link rel="apple-touch-icon" href="../../assets/icon.png">
+    <script src="./facedetect.js" type="module"></script>
+    <style>
+      img { object-fit: contain; }
+      img:hover { filter: grayscale(1); transform: scale(1.08); transition : all 0.3s ease; }   
+      @font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../../assets/lato-light.woff2') }
+      html { font-family: 'Lato', 'Segoe UI'; font-size: 24px; font-variant: small-caps; }
+      body { margin: 24px; background: black; color: white; overflow: hidden; text-align: -webkit-center; width: 100vw; height: 100vh; }
+      ::-webkit-scrollbar { height: 8px; border: 0; border-radius: 0; }
+      ::-webkit-scrollbar-thumb { background: grey }
+      ::-webkit-scrollbar-track { margin: 3px; }
+      canvas { width: 192px; height: 192px; margin: 2px; padding: 2px; cursor: grab; transform: scale(1.00); transition : all 0.3s ease; }
+      canvas:hover { filter: grayscale(1); transform: scale(1.08); transition : all 0.3s ease; }   
+    </style>
+  </head>
+  <body>
+    <component-loader></component-loader>
+    <div style="display: flex">
+      <div>
+        <div style="margin: 24px">select image to show detected faces<br>drag & drop to add your images</div>
+        <div id="images" style="display: flex; width: 98vw; overflow-x: auto; overflow-y: hidden; scroll-behavior: smooth"></div>
+      </div>
+    </div>
+    <div id="list" style="height: 10px"></div>
+    <div style="margin: 24px">hover or click on face to show details</div>
+    <div id="faces" style="overflow-y: auto"></div>
+    <div id="description" style="white-space: pre;"></div>
+  </body>
+</html>

demo/faceid/index.html

@@ -19,14 +19,23 @@
       body { margin: 0; padding: 16px; background: black; color: white; overflow-x: hidden; width: 100vw; height: 100vh; }
       body::-webkit-scrollbar { display: none; }
       .button { padding: 2px; cursor: pointer; box-shadow: 2px 2px black; width: 64px; text-align: center; place-content: center; margin-left: 16px; height: 16px; display: none }
-      .ok { position: absolute; top: 64px; right: 20px; width: 100px; background-color: grey; padding: 4px; color: black; font-size: 14px }
+      .ok { position: absolute; top: 64px; right: 20px; width: 150px; background-color: grey; padding: 4px; color: black; font-size: 14px }
     </style>
   </head>
   <body>
+    <div style="padding: 8px">
+      <h1 style="margin: 0">faceid demo using human library</h1>
+      look directly at camera and make sure that detection passes all of the required tests noted on the right hand side of the screen<br>
+      if input does not satisfies tests within specific timeout, no image will be selected<br>
+      once face image is approved, it will be compared with existing face database<br>
+      you can also store face descriptor with label in a browser's indexdb for future usage<br>
+      <br>
+      <i>note: this is not equivalent to full faceid methods as used by modern mobile phones or windows hello<br>
+        as they rely on additional infrared sensors and depth-sensing and not just camera image for additional levels of security</i>
+    </div>
     <canvas id="canvas" style="padding: 8px"></canvas>
     <canvas id="source" style="padding: 8px"></canvas>
     <video id="video" playsinline style="display: none"></video>
-    <pre id="fps" style="position: absolute; bottom: 16px; right: 20px; background-color: grey; padding: 8px; box-shadow: 2px 2px black"></pre>
     <pre id="log" style="padding: 8px"></pre>
     <div id="match" style="display: none; padding: 8px">
       <label for="name">name:</label>
@@ -34,7 +43,7 @@
       <span id="save" class="button" style="background-color: royalblue">save</span>
       <span id="delete" class="button" style="background-color: lightcoral">delete</span>
     </div>
-    <div id="retry" class="button" style="background-color: darkslategray; width: 350px; margin-top: 32px; padding: 4px">retry</div>
+    <div id="retry" class="button" style="background-color: darkslategray; width: 93%; margin-top: 32px; padding: 12px">retry</div>
     <div id="ok"></div>
   </body>
 </html>

demo/faceid/index.ts

@@ -11,11 +11,13 @@ import * as H from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human
 import * as indexDb from './indexdb'; // methods to deal with indexdb
 
 const humanConfig = { // user configuration for human, used to fine-tune behavior
+  cacheSensitivity: 0.01,
   modelBasePath: '../../models',
-  filter: { equalization: true }, // lets run with histogram equilizer
+  filter: { enabled: true, equalization: true }, // lets run with histogram equilizer
+  debug: true,
   face: {
     enabled: true,
-    detector: { rotation: true, return: true, cropFactor: 1.6, mask: false }, // return tensor is used to get detected face image
+    detector: { rotation: true, return: true, mask: false }, // return tensor is used to get detected face image
     description: { enabled: true }, // default model for face descriptor extraction is faceres
     // mobilefacenet: { enabled: true, modelPath: 'https://vladmandic.github.io/human-models/models/mobilefacenet.json' }, // alternative model
     // insightface: { enabled: true, modelPath: 'https://vladmandic.github.io/insightface/models/insightface-mobilenet-swish.json' }, // alternative model
@@ -36,28 +38,49 @@ const matchOptions = { order: 2, multiplier: 25, min: 0.2, max: 0.8 }; // for fa
 const options = {
   minConfidence: 0.6, // overal face confidence for box, face, gender, real, live
   minSize: 224, // min input to face descriptor model before degradation
-  maxTime: 10000, // max time before giving up
+  maxTime: 30000, // max time before giving up
   blinkMin: 10, // minimum duration of a valid blink
   blinkMax: 800, // maximum duration of a valid blink
   threshold: 0.5, // minimum similarity
+  distanceMin: 0.4, // closest that face is allowed to be to the cammera in cm
+  distanceMax: 1.0, // farthest that face is allowed to be to the cammera in cm
   mask: humanConfig.face.detector.mask,
   rotation: humanConfig.face.detector.rotation,
-  cropFactor: humanConfig.face.detector.cropFactor,
   ...matchOptions,
 };
 
-const ok = { // must meet all rules
-  faceCount: false,
-  faceConfidence: false,
-  facingCenter: false,
-  lookingCenter: false,
-  blinkDetected: false,
-  faceSize: false,
-  antispoofCheck: false,
-  livenessCheck: false,
-  elapsedMs: 0, // total time while waiting for valid face
+const ok: Record<string, { status: boolean | undefined, val: number }> = { // must meet all rules
+  faceCount: { status: false, val: 0 },
+  faceConfidence: { status: false, val: 0 },
+  facingCenter: { status: false, val: 0 },
+  lookingCenter: { status: false, val: 0 },
+  blinkDetected: { status: false, val: 0 },
+  faceSize: { status: false, val: 0 },
+  antispoofCheck: { status: false, val: 0 },
+  livenessCheck: { status: false, val: 0 },
+  distance: { status: false, val: 0 },
+  age: { status: false, val: 0 },
+  gender: { status: false, val: 0 },
+  timeout: { status: true, val: 0 },
+  descriptor: { status: false, val: 0 },
+  elapsedMs: { status: undefined, val: 0 }, // total time while waiting for valid face
+  detectFPS: { status: undefined, val: 0 }, // mark detection fps performance
+  drawFPS: { status: undefined, val: 0 }, // mark redraw fps performance
 };
-const allOk = () => ok.faceCount && ok.faceSize && ok.blinkDetected && ok.facingCenter && ok.lookingCenter && ok.faceConfidence && ok.antispoofCheck && ok.livenessCheck;
+
+const allOk = () => ok.faceCount.status
+  && ok.faceSize.status
+  && ok.blinkDetected.status
+  && ok.facingCenter.status
+  && ok.lookingCenter.status
+  && ok.faceConfidence.status
+  && ok.antispoofCheck.status
+  && ok.livenessCheck.status
+  && ok.distance.status
+  && ok.descriptor.status
+  && ok.age.status
+  && ok.gender.status;
+
 const current: { face: H.FaceResult | null, record: indexDb.FaceRecord | null } = { face: null, record: null }; // current face record and matched database record
 
 const blink = { // internal timers for blink start/end/duration
@@ -87,17 +110,14 @@ const dom = { // grab instances of dom objects so we dont have to look them up l
   ok: document.getElementById('ok') as HTMLDivElement,
 };
 const timestamp = { detect: 0, draw: 0 }; // holds information used to calculate performance and possible memory leaks
-const fps = { detect: 0, draw: 0 }; // holds calculated fps information for both detect and screen refresh
 let startTime = 0;
 
 const log = (...msg) => { // helper method to output messages
   dom.log.innerText += msg.join(' ') + '\n';
   console.log(...msg); // eslint-disable-line no-console
 };
-const printFPS = (msg) => dom.fps.innerText = msg; // print status element
 
 async function webCam() { // initialize webcam
-  printFPS('starting webcam...');
   // @ts-ignore resizeMode is not yet defined in tslib
   const cameraOptions: MediaStreamConstraints = { audio: false, video: { facingMode: 'user', resizeMode: 'none', width: { ideal: document.body.clientWidth } } };
   const stream: MediaStream = await navigator.mediaDevices.getUserMedia(cameraOptions);
@@ -107,6 +127,8 @@ async function webCam() { // initialize webcam
   await ready;
   dom.canvas.width = dom.video.videoWidth;
   dom.canvas.height = dom.video.videoHeight;
+  dom.canvas.style.width = '50%';
+  dom.canvas.style.height = '50%';
   if (human.env.initial) log('video:', dom.video.videoWidth, dom.video.videoHeight, '|', stream.getVideoTracks()[0].label);
   dom.canvas.onclick = () => { // pause when clicked on screen and resume on next click
     if (dom.video.paused) void dom.video.play();
@@ -119,58 +141,73 @@ async function detectionLoop() { // main detection loop
     if (current.face?.tensor) human.tf.dispose(current.face.tensor); // dispose previous tensor
     await human.detect(dom.video); // actual detection; were not capturing output in a local variable as it can also be reached via human.result
     const now = human.now();
-    fps.detect = 1000 / (now - timestamp.detect);
+    ok.detectFPS.val = Math.round(10000 / (now - timestamp.detect)) / 10;
     timestamp.detect = now;
     requestAnimationFrame(detectionLoop); // start new frame immediately
   }
 }
 
+function drawValidationTests() {
+  let y = 32;
+  for (const [key, val] of Object.entries(ok)) {
+    let el = document.getElementById(`ok-${key}`);
+    if (!el) {
+      el = document.createElement('div');
+      el.id = `ok-${key}`;
+      el.innerText = key;
+      el.className = 'ok';
+      el.style.top = `${y}px`;
+      dom.ok.appendChild(el);
+    }
+    if (typeof val.status === 'boolean') el.style.backgroundColor = val.status ? 'lightgreen' : 'lightcoral';
+    const status = val.status ? 'ok' : 'fail';
+    el.innerText = `${key}: ${val.val === 0 ? status : val.val}`;
+    y += 28;
+  }
+}
+
 async function validationLoop(): Promise<H.FaceResult> { // main screen refresh loop
   const interpolated = human.next(human.result); // smoothen result using last-known results
   human.draw.canvas(dom.video, dom.canvas); // draw canvas to screen
   await human.draw.all(dom.canvas, interpolated); // draw labels, boxes, lines, etc.
   const now = human.now();
-  fps.draw = 1000 / (now - timestamp.draw);
+  ok.drawFPS.val = Math.round(10000 / (now - timestamp.draw)) / 10;
   timestamp.draw = now;
-  printFPS(`fps: ${fps.detect.toFixed(1).padStart(5, ' ')} detect | ${fps.draw.toFixed(1).padStart(5, ' ')} draw`); // write status
-  ok.faceCount = human.result.face.length === 1; // must be exactly detected face
-  if (ok.faceCount) { // skip the rest if no face
+  ok.faceCount.val = human.result.face.length;
+  ok.faceCount.status = ok.faceCount.val === 1; // must be exactly detected face
+  if (ok.faceCount.status) { // skip the rest if no face
     const gestures: string[] = Object.values(human.result.gesture).map((gesture: H.GestureResult) => gesture.gesture); // flatten all gestures
     if (gestures.includes('blink left eye') || gestures.includes('blink right eye')) blink.start = human.now(); // blink starts when eyes get closed
     if (blink.start > 0 && !gestures.includes('blink left eye') && !gestures.includes('blink right eye')) blink.end = human.now(); // if blink started how long until eyes are back open
-    ok.blinkDetected = ok.blinkDetected || (Math.abs(blink.end - blink.start) > options.blinkMin && Math.abs(blink.end - blink.start) < options.blinkMax);
-    if (ok.blinkDetected && blink.time === 0) blink.time = Math.trunc(blink.end - blink.start);
-    ok.facingCenter = gestures.includes('facing center');
-    ok.lookingCenter = gestures.includes('looking center'); // must face camera and look at camera
-    ok.faceConfidence = (human.result.face[0].boxScore || 0) > options.minConfidence && (human.result.face[0].faceScore || 0) > options.minConfidence;
-    ok.antispoofCheck = (human.result.face[0].real || 0) > options.minConfidence;
-    ok.livenessCheck = (human.result.face[0].live || 0) > options.minConfidence;
-    ok.faceSize = human.result.face[0].box[2] >= options.minSize && human.result.face[0].box[3] >= options.minSize;
-  }
-  let y = 32;
-  for (const [key, val] of Object.entries(ok)) {
-    let el = document.getElementById(`ok-${key}`);
-    if (!el) {
-      el = document.createElement('div');
-      el.innerText = key;
-      el.className = 'ok';
-      el.style.top = `${y}px`;
-      dom.ok.appendChild(el);
-    }
-    if (typeof val === 'boolean') el.style.backgroundColor = val ? 'lightgreen' : 'lightcoral';
-    else el.innerText = `${key}:${val}`;
-    y += 28;
-  }
-  if (allOk()) { // all criteria met
-    dom.video.pause();
-    return human.result.face[0];
-  }
-  if (ok.elapsedMs > options.maxTime) { // give up
-    dom.video.pause();
-    return human.result.face[0];
+    ok.blinkDetected.status = ok.blinkDetected.status || (Math.abs(blink.end - blink.start) > options.blinkMin && Math.abs(blink.end - blink.start) < options.blinkMax);
+    if (ok.blinkDetected.status && blink.time === 0) blink.time = Math.trunc(blink.end - blink.start);
+    ok.facingCenter.status = gestures.includes('facing center');
+    ok.lookingCenter.status = gestures.includes('looking center'); // must face camera and look at camera
+    ok.faceConfidence.val = human.result.face[0].faceScore || human.result.face[0].boxScore || 0;
+    ok.faceConfidence.status = ok.faceConfidence.val >= options.minConfidence;
+    ok.antispoofCheck.val = human.result.face[0].real || 0;
+    ok.antispoofCheck.status = ok.antispoofCheck.val >= options.minConfidence;
+    ok.livenessCheck.val = human.result.face[0].live || 0;
+    ok.livenessCheck.status = ok.livenessCheck.val >= options.minConfidence;
+    ok.faceSize.val = Math.min(human.result.face[0].box[2], human.result.face[0].box[3]);
+    ok.faceSize.status = ok.faceSize.val >= options.minSize;
+    ok.distance.val = human.result.face[0].distance || 0;
+    ok.distance.status = (ok.distance.val >= options.distanceMin) && (ok.distance.val <= options.distanceMax);
+    ok.descriptor.val = human.result.face[0].embedding?.length || 0;
+    ok.descriptor.status = ok.descriptor.val > 0;
+    ok.age.val = human.result.face[0].age || 0;
+    ok.age.status = ok.age.val > 0;
+    ok.gender.val = human.result.face[0].genderScore || 0;
+    ok.gender.status = ok.gender.val >= options.minConfidence;
   }
   // run again
-  ok.elapsedMs = Math.trunc(human.now() - startTime);
+  ok.timeout.status = ok.elapsedMs.val <= options.maxTime;
+  drawValidationTests();
+  if (allOk() || !ok.timeout.status) { // all criteria met
+    dom.video.pause();
+    return human.result.face[0];
+  }
+  ok.elapsedMs.val = Math.trunc(human.now() - startTime);
   return new Promise((resolve) => {
     setTimeout(async () => {
       await validationLoop(); // run validation loop until conditions are met
@@ -198,19 +235,21 @@ async function deleteRecord() {
 }
 
 async function detectFace() {
+  dom.canvas.style.height = '';
   dom.canvas.getContext('2d')?.clearRect(0, 0, options.minSize, options.minSize);
   if (!current?.face?.tensor || !current?.face?.embedding) return false;
   console.log('face record:', current.face); // eslint-disable-line no-console
-  human.tf.browser.toPixels(current.face.tensor as unknown as H.TensorLike, dom.canvas);
+  log(`detected face: ${current.face.gender} ${current.face.age || 0}y distance ${100 * (current.face.distance || 0)}cm/${Math.round(100 * (current.face.distance || 0) / 2.54)}in`);
+  await human.draw.tensor(current.face.tensor, dom.canvas);
   if (await indexDb.count() === 0) {
-    log('face database is empty');
+    log('face database is empty: nothing to compare face with');
     document.body.style.background = 'black';
     dom.delete.style.display = 'none';
     return false;
   }
   const db = await indexDb.load();
   const descriptors = db.map((rec) => rec.descriptor).filter((desc) => desc.length > 0);
-  const res = human.match(current.face.embedding, descriptors, matchOptions);
+  const res = human.match.find(current.face.embedding, descriptors, matchOptions);
   current.record = db[res.index] || null;
   if (current.record) {
     log(`best match: ${current.record.name} | id: ${current.record.id} | similarity: ${Math.round(1000 * res.similarity) / 10}%`);
@@ -223,24 +262,27 @@ async function detectFace() {
 }
 
 async function main() { // main entry point
-  ok.faceCount = false;
-  ok.faceConfidence = false;
-  ok.facingCenter = false;
-  ok.blinkDetected = false;
-  ok.faceSize = false;
-  ok.antispoofCheck = false;
-  ok.livenessCheck = false;
-  ok.elapsedMs = 0;
+  ok.faceCount.status = false;
+  ok.faceConfidence.status = false;
+  ok.facingCenter.status = false;
+  ok.blinkDetected.status = false;
+  ok.faceSize.status = false;
+  ok.antispoofCheck.status = false;
+  ok.livenessCheck.status = false;
+  ok.age.status = false;
+  ok.gender.status = false;
+  ok.elapsedMs.val = 0;
   dom.match.style.display = 'none';
   dom.retry.style.display = 'none';
   dom.source.style.display = 'none';
+  dom.canvas.style.height = '50%';
   document.body.style.background = 'black';
   await webCam();
   await detectionLoop(); // start detection loop
   startTime = human.now();
   current.face = await validationLoop(); // start validation loop
-  dom.canvas.width = current.face.tensor?.shape[1] || options.minSize;
-  dom.canvas.height = current.face.tensor?.shape[0] || options.minSize;
+  dom.canvas.width = current.face?.tensor?.shape[1] || options.minSize;
+  dom.canvas.height = current.face?.tensor?.shape[0] || options.minSize;
   dom.source.width = dom.canvas.width;
   dom.source.height = dom.canvas.height;
   dom.canvas.style.width = '';
@@ -257,13 +299,15 @@ async function main() { // main entry point
 
 async function init() {
   log('human version:', human.version, '| tfjs version:', human.tf.version['tfjs-core']);
-  log('face embedding model:', humanConfig.face.description.enabled ? 'faceres' : '', humanConfig.face['mobilefacenet']?.enabled ? 'mobilefacenet' : '', humanConfig.face['insightface']?.enabled ? 'insightface' : '');
   log('options:', JSON.stringify(options).replace(/{|}|"|\[|\]/g, '').replace(/,/g, ' '));
-  printFPS('loading...');
-  log('known face records:', await indexDb.count());
+  log('initializing webcam...');
   await webCam(); // start webcam
+  log('loading human models...');
   await human.load(); // preload all models
-  printFPS('initializing...');
+  log('initializing human...');
+  log('face embedding model:', humanConfig.face.description.enabled ? 'faceres' : '', humanConfig.face['mobilefacenet']?.enabled ? 'mobilefacenet' : '', humanConfig.face['insightface']?.enabled ? 'insightface' : '');
+  log('loading face database...');
+  log('known face records:', await indexDb.count());
   dom.retry.addEventListener('click', main);
   dom.save.addEventListener('click', saveRecords);
   dom.delete.addEventListener('click', deleteRecord);

demo/facematch/README.md

@@ -11,7 +11,7 @@
 ## Browser Face Recognition Demo
 
 - `demo/facematch`: Demo for Browsers that uses all face description and embedding features to  
-detect, extract and identify all faces plus calculate simmilarity between them
+detect, extract and identify all faces plus calculate similarity between them
 
 It highlights functionality such as:
 
@@ -72,12 +72,13 @@ Non-linear performance that increases with number of worker threads due to commu
 
 > node node-match
 
+<!-- eslint-skip -->
 ```js
-2021-10-13 07:53:36 INFO:  options: { dbFile: './faces.json', dbMax: 10000, threadPoolSize: 6, workerSrc: './node-match-worker.js', debug: false, minThreshold: 0.9, descLength: 1024 }
-2021-10-13 07:53:36 DATA:  created shared buffer: { maxDescriptors: 10000, totalBytes: 40960000, totalElements: 10240000 }
-2021-10-13 07:53:36 DATA:  db loaded: { existingRecords: 0, newRecords: 5700 }
-2021-10-13 07:53:36 INFO:  starting worker thread pool: { totalWorkers: 6, alreadyActive: 0 }
-2021-10-13 07:53:36 STATE: submitted: { matchJobs: 100, poolSize: 6, activeWorkers: 6 }
-2021-10-13 07:53:38 STATE: { matchJobsFinished: 100, totalTimeMs: 1769, averageTimeMs: 17.69 }
-2021-10-13 07:53:38 INFO:  closing workers: { poolSize: 6, activeWorkers: 6 }
+INFO:  options: { dbFile: './faces.json', dbMax: 10000, threadPoolSize: 6, workerSrc: './node-match-worker.js', debug: false, minThreshold: 0.9, descLength: 1024 }
+DATA:  created shared buffer: { maxDescriptors: 10000, totalBytes: 40960000, totalElements: 10240000 }
+DATA:  db loaded: { existingRecords: 0, newRecords: 5700 }
+INFO:  starting worker thread pool: { totalWorkers: 6, alreadyActive: 0 }
+STATE: submitted: { matchJobs: 100, poolSize: 6, activeWorkers: 6 }
+STATE: { matchJobsFinished: 100, totalTimeMs: 1769, averageTimeMs: 17.69 }
+INFO:  closing workers: { poolSize: 6, activeWorkers: 6 }
 ```

demo/facematch/facematch.js

@@ -1,7 +1,7 @@
 /**
  * Human demo for browsers
  *
- * Demo for face descriptor analysis and face simmilarity analysis
+ * Demo for face descriptor analysis and face similarity analysis
  */
 
 /** @type {Human} */
@@ -11,7 +11,7 @@ const userConfig = {
   backend: 'humangl',
   async: true,
   warmup: 'none',
-  cacheSensitivity: 0,
+  cacheSensitivity: 0.01,
   debug: true,
   modelBasePath: '../../models/',
   deallocate: true,
@@ -22,7 +22,6 @@ const userConfig = {
   },
   face: {
     enabled: true,
-    // detector: { rotation: false, return: true, maxDetected: 50, iouThreshold: 0.206, minConfidence: 0.122 },
     detector: { return: true, rotation: true, maxDetected: 50, iouThreshold: 0.01, minConfidence: 0.2 },
     mesh: { enabled: true },
     iris: { enabled: false },
@@ -64,25 +63,16 @@ async function loadFaceMatchDB() {
   }
 }
 
-async function SelectFaceCanvas(face) {
+async function selectFaceCanvas(face) {
   // if we have face image tensor, enhance it and display it
   let embedding;
   document.getElementById('orig').style.filter = 'blur(16px)';
   if (face.tensor) {
     title('Sorting Faces by Similarity');
-    const enhanced = human.enhance(face);
-    if (enhanced) {
     const c = document.getElementById('orig');
-      const squeeze = human.tf.squeeze(enhanced);
-      const normalize = human.tf.div(squeeze, 255);
-      await human.tf.browser.toPixels(normalize, c);
-      human.tf.dispose([enhanced, squeeze, normalize]);
-      const ctx = c.getContext('2d');
-      ctx.font = 'small-caps 0.4rem "Lato"';
-      ctx.fillStyle = 'rgba(255, 255, 255, 1)';
-    }
+    await human.draw.tensor(face.tensor, c);
     const arr = db.map((rec) => rec.embedding);
-    const res = await human.match(face.embedding, arr);
+    const res = await human.match.find(face.embedding, arr);
     log('Match:', db[res.index].name);
     const emotion = face.emotion[0] ? `${Math.round(100 * face.emotion[0].score)}% ${face.emotion[0].emotion}` : 'N/A';
     document.getElementById('desc').innerHTML = `
@@ -103,11 +93,11 @@ async function SelectFaceCanvas(face) {
   for (const canvas of canvases) {
     // calculate similarity from selected face to current one in the loop
     const current = all[canvas.tag.sample][canvas.tag.face];
-    const similarity = human.similarity(face.embedding, current.embedding);
+    const similarity = human.match.similarity(face.embedding, current.embedding);
     canvas.tag.similarity = similarity;
     // get best match
     // draw the canvas
-    await human.tf.browser.toPixels(current.tensor, canvas);
+    await human.draw.tensor(current.tensor, canvas);
     const ctx = canvas.getContext('2d');
     ctx.font = 'small-caps 1rem "Lato"';
     ctx.fillStyle = 'rgba(0, 0, 0, 1)';
@@ -120,7 +110,7 @@ async function SelectFaceCanvas(face) {
     ctx.font = 'small-caps 1rem "Lato"';
     const start = human.now();
     const arr = db.map((rec) => rec.embedding);
-    const res = await human.match(current.embedding, arr);
+    const res = await human.match.find(current.embedding, arr);
     time += (human.now() - start);
     if (res.similarity > minScore) ctx.fillText(`DB: ${(100 * res.similarity).toFixed(1)}% ${db[res.index].name}`, 4, canvas.height - 30);
   }
@@ -135,7 +125,7 @@ async function SelectFaceCanvas(face) {
   title('Selected Face');
 }
 
-async function AddFaceCanvas(index, res, fileName) {
+async function addFaceCanvas(index, res, fileName) {
   all[index] = res.face;
   for (const i in res.face) {
     if (!res.face[i].tensor) continue; // did not get valid results
@@ -154,25 +144,25 @@ async function AddFaceCanvas(index, res, fileName) {
       gender: ${Math.round(100 * res.face[i].genderScore)}% ${res.face[i].gender}
       emotion: ${emotion}
     `.replace(/  /g, ' ');
-    await human.tf.browser.toPixels(res.face[i].tensor, canvas);
+    await human.draw.tensor(res.face[i].tensor, canvas);
     const ctx = canvas.getContext('2d');
-    if (!ctx) return false;
+    if (!ctx) return;
     ctx.font = 'small-caps 0.8rem "Lato"';
     ctx.fillStyle = 'rgba(255, 255, 255, 1)';
     ctx.fillText(`${res.face[i].age}y ${(100 * (res.face[i].genderScore || 0)).toFixed(1)}% ${res.face[i].gender}`, 4, canvas.height - 6);
     const arr = db.map((rec) => rec.embedding);
-    const result = human.match(res.face[i].embedding, arr);
+    const result = human.match.find(res.face[i].embedding, arr);
     ctx.font = 'small-caps 1rem "Lato"';
     if (result.similarity && res.similarity > minScore) ctx.fillText(`${(100 * result.similarity).toFixed(1)}% ${db[result.index].name}`, 4, canvas.height - 30);
     document.getElementById('faces').appendChild(canvas);
     canvas.addEventListener('click', (evt) => {
       log('Select:', 'Image:', evt.target.tag.sample, 'Face:', evt.target.tag.face, 'Source:', evt.target.tag.source, all[evt.target.tag.sample][evt.target.tag.face]);
-      SelectFaceCanvas(all[evt.target.tag.sample][evt.target.tag.face]);
+      selectFaceCanvas(all[evt.target.tag.sample][evt.target.tag.face]);
     });
   }
 }
 
-async function AddImageElement(index, image, length) {
+async function addImageElement(index, image, length) {
   const faces = all.reduce((prev, curr) => prev += curr.length, 0);
   title(`Analyzing Input Images<br> ${Math.round(100 * index / length)}% [${index} / ${length}]<br>Found ${faces} Faces`);
   return new Promise((resolve) => {
@@ -181,7 +171,7 @@ async function AddImageElement(index, image, length) {
       document.getElementById('images').appendChild(img); // and finally we can add it
       human.detect(img, userConfig)
         .then((res) => { // eslint-disable-line promise/always-return
-          AddFaceCanvas(index, res, image); // then wait until image is analyzed
+          addFaceCanvas(index, res, image); // then wait until image is analyzed
           resolve(true);
         })
         .catch(() => log('human detect error'));
@@ -222,18 +212,23 @@ async function main() {
   // could not dynamically enumerate images so using static list
   if (images.length === 0) {
     images = [
-      'ai-body.jpg', 'solvay1927.jpg', 'ai-upper.jpg',
-      'person-carolina.jpg', 'person-celeste.jpg', 'person-leila1.jpg', 'person-leila2.jpg', 'person-lexi.jpg', 'person-linda.jpg', 'person-nicole.jpg', 'person-tasia.jpg',
-      'person-tetiana.jpg', 'person-vlado1.jpg', 'person-vlado5.jpg', 'person-vlado.jpg', 'person-christina.jpg', 'person-lauren.jpg',
+      'ai-face.jpg', 'ai-upper.jpg', 'ai-body.jpg', 'solvay1927.jpg',
       'group-1.jpg', 'group-2.jpg', 'group-3.jpg', 'group-4.jpg', 'group-5.jpg', 'group-6.jpg', 'group-7.jpg',
-      'daz3d-brianna.jpg', 'daz3d-chiyo.jpg', 'daz3d-cody.jpg', 'daz3d-drew-01.jpg', 'daz3d-drew-02.jpg', 'daz3d-ella-01.jpg', 'daz3d-ella-02.jpg', 'daz3d-gillian.jpg',
-      'daz3d-hye-01.jpg', 'daz3d-hye-02.jpg', 'daz3d-kaia.jpg', 'daz3d-karen.jpg', 'daz3d-kiaria-01.jpg', 'daz3d-kiaria-02.jpg', 'daz3d-lilah-01.jpg', 'daz3d-lilah-02.jpg',
-      'daz3d-lilah-03.jpg', 'daz3d-lila.jpg', 'daz3d-lindsey.jpg', 'daz3d-megah.jpg', 'daz3d-selina-01.jpg', 'daz3d-selina-02.jpg', 'daz3d-snow.jpg',
-      'daz3d-sunshine.jpg', 'daz3d-taia.jpg', 'daz3d-tuesday-01.jpg', 'daz3d-tuesday-02.jpg', 'daz3d-tuesday-03.jpg', 'daz3d-zoe.jpg', 'daz3d-ginnifer.jpg',
-      'daz3d-_emotions01.jpg', 'daz3d-_emotions02.jpg', 'daz3d-_emotions03.jpg', 'daz3d-_emotions04.jpg', 'daz3d-_emotions05.jpg',
+      'person-celeste.jpg', 'person-christina.jpg', 'person-lauren.jpg', 'person-lexi.jpg', 'person-linda.jpg', 'person-nicole.jpg', 'person-tasia.jpg', 'person-tetiana.jpg', 'person-vlado.jpg', 'person-vlado1.jpg', 'person-vlado5.jpg',
+      'stock-group-1.jpg', 'stock-group-2.jpg',
+      'stock-models-1.jpg', 'stock-models-2.jpg', 'stock-models-3.jpg', 'stock-models-4.jpg', 'stock-models-5.jpg', 'stock-models-6.jpg', 'stock-models-7.jpg', 'stock-models-8.jpg', 'stock-models-9.jpg',
+      'stock-teen-1.jpg', 'stock-teen-2.jpg', 'stock-teen-3.jpg', 'stock-teen-4.jpg', 'stock-teen-5.jpg', 'stock-teen-6.jpg', 'stock-teen-7.jpg', 'stock-teen-8.jpg',
+      'stock-models-10.jpg', 'stock-models-11.jpg', 'stock-models-12.jpg', 'stock-models-13.jpg', 'stock-models-14.jpg', 'stock-models-15.jpg', 'stock-models-16.jpg',
+      'cgi-model-1.jpg', 'cgi-model-2.jpg', 'cgi-model-3.jpg', 'cgi-model-4.jpg', 'cgi-model-5.jpg', 'cgi-model-6.jpg', 'cgi-model-7.jpg', 'cgi-model-8.jpg', 'cgi-model-9.jpg',
+      'cgi-model-10.jpg', 'cgi-model-11.jpg', 'cgi-model-12.jpg', 'cgi-model-13.jpg', 'cgi-model-14.jpg', 'cgi-model-15.jpg', 'cgi-model-18.jpg', 'cgi-model-19.jpg',
+      'cgi-model-20.jpg', 'cgi-model-21.jpg', 'cgi-model-22.jpg', 'cgi-model-23.jpg', 'cgi-model-24.jpg', 'cgi-model-25.jpg', 'cgi-model-26.jpg', 'cgi-model-27.jpg', 'cgi-model-28.jpg', 'cgi-model-29.jpg',
+      'cgi-model-30.jpg', 'cgi-model-31.jpg', 'cgi-model-33.jpg', 'cgi-model-34.jpg',
+      'cgi-multiangle-1.jpg', 'cgi-multiangle-2.jpg', 'cgi-multiangle-3.jpg', 'cgi-multiangle-4.jpg', 'cgi-multiangle-6.jpg', 'cgi-multiangle-7.jpg', 'cgi-multiangle-8.jpg', 'cgi-multiangle-9.jpg', 'cgi-multiangle-10.jpg', 'cgi-multiangle-11.jpg',
+      'stock-emotions-a-1.jpg', 'stock-emotions-a-2.jpg', 'stock-emotions-a-3.jpg', 'stock-emotions-a-4.jpg', 'stock-emotions-a-5.jpg', 'stock-emotions-a-6.jpg', 'stock-emotions-a-7.jpg', 'stock-emotions-a-8.jpg',
+      'stock-emotions-b-1.jpg', 'stock-emotions-b-2.jpg', 'stock-emotions-b-3.jpg', 'stock-emotions-b-4.jpg', 'stock-emotions-b-5.jpg', 'stock-emotions-b-6.jpg', 'stock-emotions-b-7.jpg', 'stock-emotions-b-8.jpg',
     ];
     // add prefix for gitpages
-    images = images.map((a) => `/human/samples/in/${a}`);
+    images = images.map((a) => `../../samples/in/${a}`);
     log('Adding static image list:', images);
   } else {
     log('Discovered images:', images);
@@ -242,7 +237,7 @@ async function main() {
   // images = ['/samples/in/person-lexi.jpg', '/samples/in/person-carolina.jpg', '/samples/in/solvay1927.jpg'];
 
   const t0 = human.now();
-  for (let i = 0; i < images.length; i++) await AddImageElement(i, images[i], images.length);
+  for (let i = 0; i < images.length; i++) await addImageElement(i, images[i], images.length);
   const t1 = human.now();
 
   // print stats
@@ -256,7 +251,7 @@ async function main() {
   title('');
   log('Ready');
   human.validate(userConfig);
-  human.similarity([], []);
+  human.match.similarity([], []);
 }
 
 window.onload = main;

demo/facematch/node-match-worker.js

@@ -38,7 +38,8 @@ function match(descBuffer, options = { order: 2, multiplier: 20 }) {
     if (best < threshold || best === 0) break; // short circuit
   }
   best = (options.order === 2) ? Math.sqrt(best) : best ** (1 / options.order);
-  return { index, distance: best, similarity: Math.max(0, 100 - best) / 100.0 };
+  const similarity = Math.round(100 * Math.max(0, 100 - best) / 100.0) / 100;
+  return { index, distance: best, similarity };
 }
 
 threads.parentPort?.on('message', (msg) => {
@@ -60,11 +61,11 @@ threads.parentPort?.on('message', (msg) => {
   }
   if (typeof msg.debug !== 'undefined') { // set verbose logging
     debug = msg.debug;
-    if (debug) threads.parentPort?.postMessage(`debug: ${debug}`);
+    // if (debug) threads.parentPort?.postMessage(`debug: ${debug}`);
   }
   if (typeof msg.threshold !== 'undefined') { // set minimum similarity threshold
     threshold = msg.threshold;
-    if (debug) threads.parentPort?.postMessage(`threshold: ${threshold}`);
+    // if (debug) threads.parentPort?.postMessage(`threshold: ${threshold}`);
   }
   if (typeof msg.shutdown !== 'undefined') { // got message to close worker
     if (debug) threads.parentPort?.postMessage('shutting down');

demo/facematch/node-match.js

@@ -15,7 +15,7 @@ const options = {
   dbMax: 10000, // maximum number of records to hold in memory
   threadPoolSize: 12, // number of worker threads to create in thread pool
   workerSrc: './node-match-worker.js', // code that executes in the worker thread
-  debug: false, // verbose messages
+  debug: true, // verbose messages
   minThreshold: 0.5, // match returns first record that meets the similarity threshold, set to 0 to always scan all records
   descLength: 1024, // descriptor length
 };
@@ -176,7 +176,7 @@ async function main() {
     data.requestID++; // increase request id
     if (testOptions.fuzDescriptors) match(fuzDescriptor(descriptor)); // fuz descriptor for harder match
     else match(descriptor);
-    if (options.debug) log.info('submited job', data.requestID); // we already know what we're searching for so we can compare results
+    if (options.debug) log.debug('submited job', data.requestID); // we already know what we're searching for so we can compare results
   }
   log.state('submitted:', { matchJobs: testOptions.maxJobs, poolSize: data.workers.length, activeWorkers: data.workers.filter((worker) => !!worker).length });
 }

demo/index-pwa.js

@@ -28,10 +28,10 @@ async function updateCached(req) {
     .then((update) => {
       // update cache if request is ok
       if (update.ok) {
-        caches // eslint-disable-line promise/no-nesting
+        caches
           .open(cacheName)
           .then((cache) => cache.put(req, update))
-          .catch((err) => log('cache update error', err));
+          .catch((err) => log('cache update error', err)); // eslint-disable-line promise/no-nesting
       }
       return true;
     })
@@ -76,8 +76,8 @@ async function getCached(evt) {
 
 function cacheInit() {
   caches.open(cacheName)
-    .then((cache) => cache.addAll(cacheFiles) // eslint-disable-line promise/no-nesting
-      .then(
+    .then((cache) => cache.addAll(cacheFiles)
+      .then( // eslint-disable-line promise/no-nesting
         () => log('cache refresh:', cacheFiles.length, 'files'),
         (err) => log('cache error', err),
       ))

demo/index.html

@@ -110,10 +110,6 @@
       <canvas id="compare-canvas" width="200" height="200"></canvas>
       <div id="similarity"></div>
     </div>
-    <div id="segmentation-container" class="compare-image">
-      <canvas id="segmentation-mask" width="256" height="256" style="width: 256px; height: 256px;"></canvas>
-      <canvas id="segmentation-canvas" width="256" height="256" style="width: 256px; height: 256px;"></canvas>
-    </div>
     <div id="samples-container" class="samples-container"></div>
     <div id="hint" class="hint"></div>
     <div id="log" class="log"></div>

demo/index.js

@@ -18,7 +18,10 @@
  * ui={}: contains all variables exposed in the UI
  */
 
-// test url <https://human.local/?worker=false&async=false&bench=false&draw=true&warmup=full&backend=humangl>
+// WARNING!!!
+// This demo is written using older code style and a lot of manual setup
+// Newer versions of Human have richer functionality allowing for much cleaner & easier usage
+// It is recommended to use other demos such as `demo/typescript` for usage examples
 
 import { Human } from '../dist/human.esm.js'; // equivalent of @vladmandic/human
 import Menu from './helpers/menu.js';
@@ -83,7 +86,7 @@ const ui = {
   facing: true, // camera facing front or back
   baseBackground: 'rgba(50, 50, 50, 1)', // 'grey'
   columns: 2, // when processing sample images create this many columns
-  useWorker: true, // use web workers for processing
+  useWorker: false, // use web workers for processing
   worker: 'index-worker.js',
   maxFPSframes: 10, // keep fps history for how many frames
   modelsPreload: false, // preload human models on startup
@@ -111,7 +114,6 @@ const ui = {
   results: false, // show results tree
   lastFrame: 0, // time of last frame processing
   viewportSet: false, // internal, has custom viewport been set
-  background: null, // holds instance of segmentation background image
   transferCanvas: null, // canvas used to transfer data to and from worker
 
   // webrtc
@@ -183,7 +185,7 @@ function status(msg) {
     prevStatus = msg;
   } else {
     const video = document.getElementById('video');
-    const playing = (video.srcObject !== null) && !video.paused;
+    const playing = isLive(video) && !video.paused; // eslint-disable-line no-use-before-define
     document.getElementById('play').style.display = playing ? 'none' : 'block';
     document.getElementById('loader').style.display = 'none';
     div.innerText = '';
@@ -193,7 +195,6 @@ function status(msg) {
 async function videoPlay(videoElement = document.getElementById('video')) {
   document.getElementById('btnStartText').innerHTML = 'pause video';
   await videoElement.play();
-  // status();
 }
 
 async function videoPause() {
@@ -221,21 +222,13 @@ async function calcSimmilarity(result) {
     compare.original = result;
     log('setting face compare baseline:', result.face[0]);
     if (result.face[0].tensor) {
-      const enhanced = human.enhance(result.face[0]);
-      if (enhanced) {
       const c = document.getElementById('orig');
-        const squeeze = human.tf.squeeze(enhanced);
-        const norm = human.tf.div(squeeze, 255);
-        human.tf.browser.toPixels(norm, c);
-        human.tf.dispose(enhanced);
-        human.tf.dispose(squeeze);
-        human.tf.dispose(norm);
-      }
+      human.draw.tensor(result.face[0].tensor, c);
     } else {
       document.getElementById('compare-canvas').getContext('2d').drawImage(compare.original.canvas, 0, 0, 200, 200);
     }
   }
-  const similarity = human.similarity(compare.original.face[0].embedding, result.face[0].embedding);
+  const similarity = human.match.similarity(compare.original.face[0].embedding, result.face[0].embedding);
   document.getElementById('similarity').innerText = `similarity: ${Math.trunc(1000 * similarity) / 10}%`;
 }
 
@@ -263,21 +256,7 @@ async function drawResults(input) {
   // draw fps chart
   await menu.process.updateChart('FPS', ui.detectFPS);
 
-  document.getElementById('segmentation-container').style.display = userConfig.segmentation.enabled ? 'block' : 'none';
-  if (userConfig.segmentation.enabled && ui.buffered) { // refresh segmentation if using buffered output
-    const seg = await human.segmentation(input, ui.background);
-    if (seg.alpha) {
-      const canvasSegMask = document.getElementById('segmentation-mask');
-      const ctxSegMask = canvasSegMask.getContext('2d');
-      ctxSegMask.clearRect(0, 0, canvasSegMask.width, canvasSegMask.height); // need to clear as seg.alpha is alpha based canvas so it adds
-      ctxSegMask.drawImage(seg.alpha, 0, 0, seg.alpha.width, seg.alpha.height, 0, 0, canvasSegMask.width, canvasSegMask.height);
-      const canvasSegCanvas = document.getElementById('segmentation-canvas');
-      const ctxSegCanvas = canvasSegCanvas.getContext('2d');
-      ctxSegCanvas.clearRect(0, 0, canvasSegCanvas.width, canvasSegCanvas.height); // need to clear as seg.alpha is alpha based canvas so it adds
-      ctxSegCanvas.drawImage(seg.canvas, 0, 0, seg.alpha.width, seg.alpha.height, 0, 0, canvasSegCanvas.width, canvasSegCanvas.height);
-    }
-    // result.canvas = seg.alpha;
-  } else if (!result.canvas || ui.buffered) { // refresh with input if using buffered output or if missing canvas
+  if (!result.canvas || ui.buffered) { // refresh with input if using buffered output or if missing canvas
     const image = await human.image(input, false);
     result.canvas = image.canvas;
     human.tf.dispose(image.tensor);
@@ -534,17 +513,6 @@ function runHumanDetect(input, canvas, timestamp) {
     human.detect(input, userConfig)
       .then((result) => {
         status();
-        /*
-        setTimeout(async () => { // simulate gl context lost 2sec after initial detection
-          const ext = human.gl && human.gl.gl ? human.gl.gl.getExtension('WEBGL_lose_context') : {};
-          if (ext && ext.loseContext) {
-            log('simulate context lost:', human.env.webgl, human.gl, ext);
-            human.gl.gl.getExtension('WEBGL_lose_context').loseContext();
-            await videoPause();
-            status('Exception: WebGL');
-          }
-        }, 2000);
-        */
         if (result.performance && result.performance.total) ui.detectFPS.push(1000 / result.performance.total);
         if (ui.detectFPS.length > ui.maxFPSframes) ui.detectFPS.shift();
         if (ui.bench) {
@@ -638,21 +606,15 @@ async function processImage(input, title) {
 
 async function processVideo(input, title) {
   status(`processing video: ${title}`);
-  // const video = document.getElementById('video-file') || document.createElement('video');
   const video = document.getElementById('video');
   const canvas = document.getElementById('canvas');
-  // video.id = 'video-file';
-  // video.controls = true;
-  // video.loop = true;
-  // video.style.display = 'none';
-  // document.body.appendChild(video);
   video.addEventListener('error', () => status(`video loading error: ${video.error.message}`));
   video.addEventListener('canplay', async () => {
     for (const m of Object.values(menu)) m.hide();
     document.getElementById('samples-container').style.display = 'none';
     canvas.style.display = 'block';
     await videoPlay();
-    if (!ui.detectThread) runHumanDetect(video, canvas);
+    runHumanDetect(video, canvas);
   });
   video.srcObject = null;
   video.src = input;
@@ -665,9 +627,8 @@ async function detectVideo() {
   const canvas = document.getElementById('canvas');
   canvas.style.display = 'block';
   cancelAnimationFrame(ui.detectThread);
-  if ((video.srcObject !== null) && !video.paused) {
+  if (isLive(video) && !video.paused) {
     await videoPause();
-    // if (ui.drawThread) cancelAnimationFrame(ui.drawThread);
   } else {
     const cameraError = await setupCamera();
     if (!cameraError) {
@@ -743,7 +704,6 @@ function setupMenu() {
   menu.image.addBool('technicolor', userConfig.filter, 'technicolor', (val) => userConfig.filter.technicolor = val);
   menu.image.addBool('polaroid', userConfig.filter, 'polaroid', (val) => userConfig.filter.polaroid = val);
   menu.image.addHTML('<input type="file" id="file-input" class="input-file"></input> &nbsp input');
-  menu.image.addHTML('<input type="file" id="file-background" class="input-file"></input> &nbsp background');
 
   menu.process = new Menu(document.body, '', { top, left: x[2] });
   menu.process.addList('backend', ['cpu', 'webgl', 'wasm', 'humangl'], userConfig.backend, (val) => userConfig.backend = val);
@@ -791,8 +751,6 @@ function setupMenu() {
   menu.models.addHTML('<hr style="border-style: inset; border-color: dimgray">');
   menu.models.addBool('gestures', userConfig.gesture, 'enabled', (val) => userConfig.gesture.enabled = val);
   menu.models.addHTML('<hr style="border-style: inset; border-color: dimgray">');
-  menu.models.addBool('body segmentation', userConfig.segmentation, 'enabled', (val) => userConfig.segmentation.enabled = val);
-  menu.models.addHTML('<hr style="border-style: inset; border-color: dimgray">');
   menu.models.addBool('object detection', userConfig.object, 'enabled', (val) => userConfig.object.enabled = val);
   menu.models.addHTML('<hr style="border-style: inset; border-color: dimgray">');
   menu.models.addBool('face compare', compare, 'enabled', (val) => {
@@ -812,6 +770,7 @@ function setupMenu() {
 
 async function resize() {
   window.onresize = null;
+  log('resize');
   // best setting for mobile, ignored for desktop
   // can set dynamic value such as Math.min(1, Math.round(100 * window.innerWidth / 960) / 100);
   const viewportScale = 0.7;
@@ -860,42 +819,12 @@ async function processDataURL(f, action) {
         if (e.target.result.startsWith('data:video')) await processVideo(e.target.result, f.name);
         document.getElementById('canvas').style.display = 'none';
       }
-      if (action === 'background') {
-        const image = new Image();
-        image.onerror = async () => status('image loading error');
-        image.onload = async () => {
-          ui.background = image;
-          if (document.getElementById('canvas').style.display === 'block') { // replace canvas used for video
-            const canvas = document.getElementById('canvas');
-            const ctx = canvas.getContext('2d');
-            const seg = await human.segmentation(canvas, ui.background, userConfig);
-            if (seg.canvas) ctx.drawImage(seg.canvas, 0, 0);
-          } else {
-            const canvases = document.getElementById('samples-container').children; // replace loaded images
-            for (const canvas of canvases) {
-              const ctx = canvas.getContext('2d');
-              const seg = await human.segmentation(canvas, ui.background, userConfig);
-              if (seg.canvas) ctx.drawImage(seg.canvas, 0, 0);
-            }
-          }
-        };
-        image.src = e.target.result;
-      }
       resolve(true);
     };
     reader.readAsDataURL(f);
   });
 }
 
-async function runSegmentation() {
-  document.getElementById('file-background').onchange = async (evt) => {
-    userConfig.segmentation.enabled = true;
-    evt.preventDefault();
-    if (evt.target.files.length < 2) ui.columns = 1;
-    for (const f of evt.target.files) await processDataURL(f, 'background');
-  };
-}
-
 async function dragAndDrop() {
   document.body.addEventListener('dragenter', (evt) => evt.preventDefault());
   document.body.addEventListener('dragleave', (evt) => evt.preventDefault());
@@ -1048,8 +977,7 @@ async function main() {
   if (ui.modelsPreload && !ui.useWorker) {
     status('loading');
     await human.load(userConfig); // this is not required, just pre-loads all models
-    const loaded = Object.keys(human.models).filter((a) => human.models[a]);
-    log('demo loaded models:', loaded);
+    log('demo loaded models:', human.models.loaded());
   } else {
     await human.init();
   }
@@ -1071,9 +999,6 @@ async function main() {
   // init drag & drop
   await dragAndDrop();
 
-  // init segmentation
-  await runSegmentation();
-
   if (params.has('image')) {
     try {
       const image = JSON.parse(params.get('image'));

demo/multithread/README.md

@@ -16,6 +16,7 @@
 node demo/nodejs/node-multiprocess.js
 ```
 
+<!-- eslint-skip -->
 ```json
 2021-06-01 08:54:19 INFO:  @vladmandic/human version 2.0.0
 2021-06-01 08:54:19 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0

demo/multithread/index.html

@@ -9,10 +9,10 @@
     <meta name="description" content="Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition; Author: Vladimir Mandic <https://github.com/vladmandic>">
     <meta name="msapplication-tooltip" content="Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition; Author: Vladimir Mandic <https://github.com/vladmandic>">
     <meta name="theme-color" content="#000000">
-    <link rel="manifest" href="../manifest.webmanifest">
+    <link rel="manifest" href="../../manifest.webmanifest">
     <link rel="shortcut icon" href="../../favicon.ico" type="image/x-icon">
     <link rel="apple-touch-icon" href="../../assets/icon.png">
-    <script src="./index.js" type="module"></script>
+    <script src="../multithread/index.js" type="module"></script>
     <style>
       @font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../../assets/lato-light.woff2') }
       html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }

demo/multithread/index.js

@@ -8,7 +8,7 @@
 import { Human } from '../../dist/human.esm.js'; // equivalent of @vladmandic/human
 import GLBench from '../helpers/gl-bench.js';
 
-const workerJS = './worker.js';
+const workerJS = '../multithread/worker.js';
 
 const config = {
   main: { // processes input and runs gesture analysis

demo/multithread/node-multiprocess.js

@@ -8,12 +8,12 @@
 
 const fs = require('fs');
 const path = require('path');
-const log = require('@vladmandic/pilogger'); // eslint-disable-line node/no-unpublished-require
 const childProcess = require('child_process'); // eslint-disable-line camelcase
+const log = require('@vladmandic/pilogger'); // eslint-disable-line node/no-unpublished-require
 // note that main process does not import human or tfjs at all, it's all done from worker process
 
 const workerFile = 'demo/multithread/node-multiprocess-worker.js';
-const imgPathRoot = './assets'; // modify to include your sample images
+const imgPathRoot = './samples/in'; // modify to include your sample images
 const numWorkers = 4; // how many workers will be started
 const workers = []; // this holds worker processes
 const images = []; // this holds queue of enumerated images
@@ -22,7 +22,7 @@ let numImages;
 
 // trigered by main when worker sends ready message
 // if image pool is empty, signal worker to exit otherwise dispatch image to worker and remove image from queue
-async function detect(worker) {
+async function submitDetect(worker) {
   if (!t[2]) t[2] = process.hrtime.bigint(); // first time do a timestamp so we can measure initial latency
   if (images.length === numImages) worker.send({ test: true }); // for first image in queue just measure latency
   if (images.length === 0) worker.send({ exit: true }); // nothing left in queue
@@ -79,7 +79,7 @@ async function main() {
     // if message is processing result, just print how many faces were detected
     // otherwise it's an unknown message
     workers[i].on('message', (msg) => {
-      if (msg.ready) detect(workers[i]);
+      if (msg.ready) submitDetect(workers[i]);
       else if (msg.image) log.data('Main: worker finished:', workers[i].pid, 'detected faces:', msg.detected.face?.length, 'bodies:', msg.detected.body?.length, 'hands:', msg.detected.hand?.length, 'objects:', msg.detected.object?.length);
       else if (msg.test) measureLatency();
       else log.data('Main: worker message:', workers[i].pid, msg);

demo/nodejs/README.md

@@ -28,7 +28,8 @@ or you can pass a path to image to analyze, either on local filesystem or using
 node demo/nodejs/node.js
 ```
 
-```json
+<!-- eslint-skip -->
+```js
 2021-06-01 08:52:15 INFO:  @vladmandic/human version 2.0.0
 2021-06-01 08:52:15 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
 2021-06-01 08:52:15 INFO:  Current folder: /home/vlado/dev/human
@@ -82,7 +83,7 @@ node demo/nodejs/node.js
     detector: { modelPath: 'handdetect.json' },
     skeleton: { modelPath: 'handskeleton.json' }
   },
-  object: { enabled: true, modelPath: 'mb3-centernet.json', minConfidence: 0.2, iouThreshold: 0.4, maxDetected: 10, skipFrames: 19 }
+  object: { enabled: true, modelPath: 'centernet.json', minConfidence: 0.2, iouThreshold: 0.4, maxDetected: 10, skipFrames: 19 }
 }
 08:52:15.673 Human: version: 2.0.0
 08:52:15.674 Human: tfjs version: 3.6.0
@@ -96,7 +97,7 @@ node demo/nodejs/node.js
 08:52:15.847 Human: load model: file://models/handdetect.json
 08:52:15.847 Human: load model: file://models/handskeleton.json
 08:52:15.914 Human: load model: file://models/movenet-lightning.json
-08:52:15.957 Human: load model: file://models/mb3-centernet.json
+08:52:15.957 Human: load model: file://models/centernet.json
 08:52:16.015 Human: load model: file://models/faceres.json
 08:52:16.015 Human: tf engine state: 50796152 bytes 1318 tensors
 2021-06-01 08:52:16 INFO:  Loaded: [ 'face', 'movenet', 'handpose', 'emotion', 'centernet', 'faceres', [length]: 6 ]

demo/nodejs/node-age-gender.js

@@ -0,0 +1,70 @@
+const fs = require('fs');
+const process = require('process');
+const log = require('@vladmandic/pilogger'); // eslint-disable-line node/no-unpublished-require
+const tf = require('@tensorflow/tfjs-node'); // eslint-disable-line node/no-unpublished-require
+const Human = require('../../dist/human.node.js');
+
+const humanConfig = {
+  debug: false,
+  face: {
+    enabled: true,
+    detector: { enabled: true, modelPath: 'blazeface.json' },
+    description: { enabled: true, modelPath: 'faceres.json' },
+    // gear: { enabled: true, modelPath: '/home/vlado/dev/human-models/models/gear.json' },
+    // ssrnet: { enabled: true, modelPathAge: '/home/vlado/dev/human-models/models/age.json', modelPathGender: '/home/vlado/dev/human-models/models/gender.json' },
+    emotion: { enabled: false },
+    mesh: { enabled: false },
+    iris: { enabled: false },
+    antispoof: { enabled: false },
+    liveness: { enabled: false },
+  },
+  body: { enabled: false },
+  hand: { enabled: false },
+  gesture: { enabled: false },
+};
+const human = new Human.Human(humanConfig);
+const ageThreshold = 18;
+
+async function detect(inputFile) {
+  try {
+    const buffer = fs.readFileSync(inputFile);
+    const tensor = human.tf.node.decodeImage(buffer);
+    const result = await human.detect(tensor);
+    human.tf.dispose(tensor);
+    if (!result || !result.face || result.face.length === 0) return false;
+    let msg = ` file=${inputFile} resolution=${tensor.shape}`;
+    for (const face of result.face) {
+      msg = ` file=${inputFile} resolution=${tensor.shape} age=${face.age} gender=${face.gender} confidence=${face.genderScore}`;
+      if (face.age < ageThreshold) {
+        log.warn('fail:' + msg);
+        return true;
+      }
+    }
+    log.info('pass: ' + msg);
+    return false;
+  } catch (err) {
+    log.error(`error: file=${inputFile}: ${err}`);
+  }
+  return false;
+}
+
+async function main() {
+  log.info(`Human: version=${human.version} tf=${tf.version_core}`);
+  process.noDeprecation = true;
+  if (process.argv.length < 3) return;
+  await human.load();
+  await human.warmup();
+  const t0 = performance.now();
+  const args = process.argv.slice(2);
+  let pass = 0;
+  let fail = 0;
+  for (const arg of args) {
+    const ok = await detect(arg);
+    if (ok) pass++;
+    else fail++;
+  }
+  const t1 = performance.now();
+  log.info(`Human: files=${args.length} pass=${pass} fail=${fail} time=${Math.round(t1 - t0)} fps=${Math.round(10000 * args.length / (t1 - t0)) / 10}`);
+}
+
+main();

demo/nodejs/node-bench.js

@@ -0,0 +1,66 @@
+/**
+ * Human simple demo for NodeJS
+ */
+
+const childProcess = require('child_process'); // eslint-disable-line camelcase
+const log = require('@vladmandic/pilogger'); // eslint-disable-line node/no-unpublished-require
+const canvas = require('canvas'); // eslint-disable-line node/no-unpublished-require
+
+const config = {
+  cacheSensitivity: 0.01,
+  wasmPlatformFetch: true,
+  modelBasePath: 'https://vladmandic.github.io/human-models/models/',
+};
+const count = 10;
+
+async function loadImage(input) {
+  const inputImage = await canvas.loadImage(input);
+  const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height);
+  const inputCtx = inputCanvas.getContext('2d');
+  inputCtx.drawImage(inputImage, 0, 0);
+  const imageData = inputCtx.getImageData(0, 0, inputCanvas.width, inputCanvas.height);
+  process.send({ input, resolution: [inputImage.width, inputImage.height] });
+  return imageData;
+}
+
+async function runHuman(module, backend) {
+  if (backend === 'wasm') require('@tensorflow/tfjs-backend-wasm'); // eslint-disable-line node/no-unpublished-require, global-require
+  const Human = require('../../dist/' + module); // eslint-disable-line global-require, import/no-dynamic-require
+  config.backend = backend;
+  const human = new Human.Human(config);
+  human.env.Canvas = canvas.Canvas;
+  human.env.Image = canvas.Image;
+  human.env.ImageData = canvas.ImageData;
+  process.send({ human: human.version, module });
+  await human.init();
+  process.send({ desired: human.config.backend, wasm: human.env.wasm, tfjs: human.tf.version.tfjs, tensorflow: human.env.tensorflow });
+  const imageData = await loadImage('samples/in/ai-body.jpg');
+  const t0 = human.now();
+  await human.load();
+  const t1 = human.now();
+  await human.warmup();
+  const t2 = human.now();
+  for (let i = 0; i < count; i++) await human.detect(imageData);
+  const t3 = human.now();
+  process.send({ backend: human.tf.getBackend(), load: Math.round(t1 - t0), warmup: Math.round(t2 - t1), detect: Math.round(t3 - t2), count, memory: human.tf.memory().numBytes });
+}
+
+async function executeWorker(args) {
+  return new Promise((resolve) => {
+    const worker = childProcess.fork(process.argv[1], args);
+    worker.on('message', (msg) => log.data(msg));
+    worker.on('exit', () => resolve(true));
+  });
+}
+
+async function main() {
+  if (process.argv[2]) {
+    await runHuman(process.argv[2], process.argv[3]);
+  } else {
+    await executeWorker(['human.node.js', 'tensorflow']);
+    await executeWorker(['human.node-gpu.js', 'tensorflow']);
+    await executeWorker(['human.node-wasm.js', 'wasm']);
+  }
+}
+
+main();

demo/nodejs/node-canvas.js

@@ -15,8 +15,8 @@ const Human = require('../../dist/human.node.js'); // use this when using human
 
 const config = { // just enable all and leave default settings
   debug: false,
-  face: { enabled: true }, // includes mesh, iris, emotion, descriptor
-  hand: { enabled: true, maxDetected: 2, minConfidence: 0.5, detector: { modelPath: 'handtrack.json' } }, // use alternative hand model
+  face: { enabled: true, detector: { maxDetected: 10 } }, // includes mesh, iris, emotion, descriptor
+  hand: { enabled: true, maxDetected: 20, minConfidence: 0.5, detector: { modelPath: 'handtrack.json' } }, // use alternative hand model
   body: { enabled: true },
   object: { enabled: true },
   gestures: { enabled: true },
@@ -24,6 +24,7 @@ const config = { // just enable all and leave default settings
 
 async function main() {
   log.header();
+  process.noDeprecation = true;
 
   globalThis.Canvas = canvas.Canvas; // patch global namespace with canvas library
   globalThis.ImageData = canvas.ImageData; // patch global namespace with canvas library
@@ -35,12 +36,13 @@ async function main() {
   log.info('Human:', human.version, 'TF:', tf.version_core);
 
   await human.load(); // pre-load models
-  log.info('Loaded models:', Object.keys(human.models).filter((a) => human.models[a]));
+  log.info('Loaded models:', human.models.loaded());
   log.info('Memory state:', human.tf.engine().memory());
 
   // parse cmdline
   const input = process.argv[2];
-  const output = process.argv[3];
+  let output = process.argv[3];
+  if (!output.toLowerCase().endsWith('.jpg')) output += '.jpg';
   if (process.argv.length !== 4) log.error('Parameters: <input-image> <output-image> missing');
   else if (!fs.existsSync(input) && !input.startsWith('http')) log.error(`File not found: ${process.argv[2]}`);
   else {
@@ -54,9 +56,6 @@ async function main() {
 
     // run detection
     const result = await human.detect(imageData);
-    // run segmentation
-    // const seg = await human.segmentation(inputCanvas);
-    // log.data('Segmentation:', { data: seg.data.length, alpha: typeof seg.alpha, canvas: typeof seg.canvas });
 
     // print results summary
     const persons = result.persons; // invoke persons getter, only used to print summary on console

demo/nodejs/node-event.js

@@ -5,8 +5,6 @@
 const fs = require('fs');
 const process = require('process');
 
-let fetch; // fetch is dynamically imported later
-
 const log = require('@vladmandic/pilogger'); // eslint-disable-line node/no-unpublished-require
 // in nodejs environments tfjs-node is required to be loaded before human
 const tf = require('@tensorflow/tfjs-node'); // eslint-disable-line node/no-unpublished-require
@@ -38,13 +36,13 @@ async function detect(input) {
   let buffer;
   log.info('Loading image:', input);
   if (input.startsWith('http:') || input.startsWith('https:')) {
-    fetch = (await import('node-fetch')).default; // eslint-disable-line node/no-unpublished-import
     const res = await fetch(input);
-    if (res && res.ok) buffer = await res.buffer();
+    if (res && res.ok) buffer = Buffer.from(await res.arrayBuffer());
     else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type'));
   } else {
     buffer = fs.readFileSync(input);
   }
+  log.data('Image bytes:', buffer?.length, 'buffer:', buffer?.slice(0, 32));
 
   // decode image using tfjs-node so we don't need external depenencies
   if (!buffer) return;
@@ -67,8 +65,7 @@ async function main() {
     });
 
     human.events.addEventListener('load', () => {
-      const loaded = Object.keys(human.models).filter((a) => human.models[a]);
-      log.info('Event Loaded:', loaded, human.tf.engine().memory());
+      log.info('Event Loaded:', human.models.loaded(), human.tf.engine().memory());
     });
 
     human.events.addEventListener('image', () => {
@@ -80,7 +77,7 @@ async function main() {
       const persons = human.result.persons;
       for (let i = 0; i < persons.length; i++) {
         const face = persons[i].face;
-        const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.iris}` : null;
+        const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.distance}` : null;
         const body = persons[i].body;
         const bodyTxt = body ? `score:${body.score} keypoints:${body.keypoints?.length}` : null;
         log.data(`  #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`);

demo/nodejs/node-fetch.js

@@ -16,8 +16,7 @@ const humanConfig = {
 };
 
 async function main(inputFile) {
-  // @ts-ignore
-  global.fetch = (await import('node-fetch')).default; // eslint-disable-line node/no-unpublished-import
+  global.fetch = (await import('node-fetch')).default; // eslint-disable-line node/no-unpublished-import, import/no-unresolved, node/no-missing-import, node/no-extraneous-import
   const human = new Human.Human(humanConfig); // create instance of human using default configuration
   log.info('Human:', human.version, 'TF:', tf.version_core);
   await human.load(); // optional as models would be loaded on-demand first time they are required

demo/nodejs/node-similarity.js

@@ -27,8 +27,7 @@ async function init() {
   await human.tf.ready();
   log.info('Human:', human.version, 'TF:', tf.version_core);
   await human.load();
-  const loaded = Object.keys(human.models).filter((a) => human.models[a]);
-  log.info('Loaded:', loaded);
+  log.info('Loaded:', human.models.loaded());
   log.info('Memory state:', human.tf.engine().memory());
 }
 
@@ -46,10 +45,12 @@ async function detect(input) {
 }
 
 async function main() {
+  process.noDeprecation = true;
   log.configure({ inspect: { breakLength: 265 } });
   log.header();
   if (process.argv.length !== 4) {
-    throw new Error('Parameters: <first image> <second image> missing');
+    log.error('Parameters: <first image> <second image> missing');
+    return;
   }
   await init();
   const res1 = await detect(process.argv[2]);
@@ -57,7 +58,7 @@ async function main() {
   if (!res1 || !res1.face || res1.face.length === 0 || !res2 || !res2.face || res2.face.length === 0) {
     throw new Error('Could not detect face descriptors');
   }
-  const similarity = human.similarity(res1.face[0].embedding, res2.face[0].embedding, { order: 2 });
+  const similarity = human.match.similarity(res1.face[0].embedding, res2.face[0].embedding, { order: 2 });
   log.data('Similarity: ', similarity);
 }
 

demo/nodejs/node-simple.js

@@ -12,9 +12,12 @@ const Human = require('../../dist/human.node.js'); // use this when using human
 
 const humanConfig = {
   // add any custom config here
+  debug: true,
+  body: { enabled: false },
 };
 
 async function detect(inputFile) {
+  process.noDeprecation = true;
   const human = new Human.Human(humanConfig); // create instance of human using default configuration
   console.log('Human:', human.version, 'TF:', tf.version_core); // eslint-disable-line no-console
   await human.load(); // optional as models would be loaded on-demand first time they are required

demo/nodejs/node-video.js

@@ -11,10 +11,11 @@
  * Working version of `ffmpeg` must be present on the system
 */
 
+const process = require('process');
 const spawn = require('child_process').spawn;
 const log = require('@vladmandic/pilogger'); // eslint-disable-line node/no-unpublished-require
 // in nodejs environments tfjs-node is required to be loaded before human
-const tf = require('@tensorflow/tfjs-node'); // eslint-disable-line node/no-unpublished-require
+// const tf = require('@tensorflow/tfjs-node'); // eslint-disable-line node/no-unpublished-require
 // const human = require('@vladmandic/human'); // use this when human is installed as module (majority of use cases)
 const Pipe2Jpeg = require('pipe2jpeg'); // eslint-disable-line node/no-missing-require, import/no-unresolved
 // const human = require('@vladmandic/human'); // use this when human is installed as module (majority of use cases)
@@ -22,7 +23,8 @@ const Human = require('../../dist/human.node.js'); // use this when using human
 
 let count = 0; // counter
 let busy = false; // busy flag
-const inputFile = './test.mp4';
+let inputFile = './test.mp4';
+if (process.argv.length === 3) inputFile = process.argv[2];
 
 const humanConfig = {
   modelBasePath: 'file://models/',
@@ -59,24 +61,27 @@ const ffmpegParams = [
   'pipe:1', // output to unix pipe that is then captured by pipe2jpeg
 ];
 
-async function process(jpegBuffer) {
+async function detect(jpegBuffer) {
   if (busy) return; // skip processing if busy
   busy = true;
   const tensor = human.tf.node.decodeJpeg(jpegBuffer, 3); // decode jpeg buffer to raw tensor
-  log.state('input frame:', ++count, 'size:', jpegBuffer.length, 'decoded shape:', tensor.shape);
   const res = await human.detect(tensor);
-  log.data('gesture', JSON.stringify(res.gesture));
-  // do processing here
-  tf.dispose(tensor); // must dispose tensor
+  human.tf.dispose(tensor); // must dispose tensor
+  // start custom processing here
+  log.data('frame', { frame: ++count, size: jpegBuffer.length, shape: tensor.shape, face: res?.face?.length, body: res?.body?.length, hand: res?.hand?.length, gesture: res?.gesture?.length });
+  if (res?.face?.[0]) log.data('person', { score: [res.face[0].boxScore, res.face[0].faceScore], age: res.face[0].age || 0, gender: [res.face[0].genderScore || 0, res.face[0].gender], emotion: res.face[0].emotion?.[0] });
+  // at the of processing mark loop as not busy so it can process next frame
   busy = false;
 }
 
 async function main() {
   log.header();
+  process.noDeprecation = true;
   await human.tf.ready();
   // pre-load models
-  log.info('human:', human.version, 'tf:', tf.version_core);
-  pipe2jpeg.on('jpeg', (jpegBuffer) => process(jpegBuffer));
+  log.info({ human: human.version, tf: human.tf.version_core });
+  log.info({ input: inputFile });
+  pipe2jpeg.on('data', (jpegBuffer) => detect(jpegBuffer));
 
   const ffmpeg = spawn('ffmpeg', ffmpegParams, { stdio: ['ignore', 'pipe', 'ignore'] });
   ffmpeg.on('error', (error) => log.error('ffmpeg error:', error));

demo/nodejs/node.js

@@ -1,15 +1,11 @@
 /**
  * Human demo for NodeJS
- *
- * Requires [node-fetch](https://www.npmjs.com/package/node-fetch) to provide `fetch` functionality in NodeJS environment
 */
 
-const log = require('@vladmandic/pilogger'); // eslint-disable-line node/no-unpublished-require
 const fs = require('fs');
 const path = require('path');
 const process = require('process');
-
-let fetch; // fetch is dynamically imported later
+const log = require('@vladmandic/pilogger'); // eslint-disable-line node/no-unpublished-require
 
 // in nodejs environments tfjs-node is required to be loaded before human
 const tf = require('@tensorflow/tfjs-node'); // eslint-disable-line node/no-unpublished-require
@@ -53,8 +49,7 @@ async function init() {
   log.info('Human:', human.version);
   // log.info('Active Configuration', human.config);
   await human.load();
-  const loaded = Object.keys(human.models).filter((a) => human.models[a]);
-  log.info('Loaded:', loaded);
+  log.info('Loaded:', human.models.loaded());
   // log.info('Memory state:', human.tf.engine().memory());
   log.data(tf.backend().binding ? tf.backend().binding.TF_Version : null);
 }
@@ -65,11 +60,12 @@ async function detect(input) {
   log.info('Loading image:', input);
   if (input.startsWith('http:') || input.startsWith('https:')) {
     const res = await fetch(input);
-    if (res && res.ok) buffer = await res.buffer();
+    if (res && res.ok) buffer = Buffer.from(await res.arrayBuffer());
     else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type'));
   } else {
     buffer = fs.readFileSync(input);
   }
+  log.data('Image bytes:', buffer?.length, 'buffer:', buffer?.slice(0, 32));
 
   // decode image using tfjs-node so we don't need external depenencies
   // can also be done using canvas.js or some other 3rd party image library
@@ -96,7 +92,7 @@ async function detect(input) {
   try {
     result = await human.detect(tensor, myConfig);
   } catch (err) {
-    log.error('caught');
+    log.error('caught', err);
   }
 
   // dispose image tensor as we no longer need it
@@ -108,7 +104,7 @@ async function detect(input) {
     for (let i = 0; i < result.face.length; i++) {
       const face = result.face[i];
       const emotion = face.emotion.reduce((prev, curr) => (prev.score > curr.score ? prev : curr));
-      log.data(`  Face: #${i} boxScore:${face.boxScore} faceScore:${face.faceScore} age:${face.age} genderScore:${face.genderScore} gender:${face.gender} emotionScore:${emotion.score} emotion:${emotion.emotion} iris:${face.iris}`);
+      log.data(`  Face: #${i} boxScore:${face.boxScore} faceScore:${face.faceScore} age:${face.age} genderScore:${face.genderScore} gender:${face.gender} emotionScore:${emotion.score} emotion:${emotion.emotion} distance:${face.distance}`);
     }
   } else {
     log.data('  Face: N/A');
@@ -191,8 +187,8 @@ async function test() {
 async function main() {
   log.configure({ inspect: { breakLength: 265 } });
   log.header();
+  process.noDeprecation = true;
   log.info('Current folder:', process.env.PWD);
-  fetch = (await import('node-fetch')).default; // eslint-disable-line node/no-unpublished-import
   await init();
   const f = process.argv[2];
   if (process.argv.length !== 3) {

demo/nodejs/process-folder.js

@@ -13,54 +13,52 @@ const process = require('process');
 const log = require('@vladmandic/pilogger'); // eslint-disable-line node/no-unpublished-require
 const canvas = require('canvas'); // eslint-disable-line node/no-unpublished-require
 // for nodejs, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
-const tf = require('@tensorflow/tfjs-node'); // eslint-disable-line node/no-unpublished-require
+const tf = require('@tensorflow/tfjs-node-gpu'); // eslint-disable-line node/no-unpublished-require
 const Human = require('../../dist/human.node-gpu.js'); // this is 'const Human = require('../dist/human.node-gpu.js').default;'
 
 const config = { // just enable all and leave default settings
+  modelBasePath: 'file://models',
   debug: true,
-  async: false,
-  cacheSensitivity: 0,
-  face: { enabled: true, detector: { maxDetected: 20 } },
-  object: { enabled: true },
+  softwareKernels: true, // slower but enhanced precision since face rotation can work in software mode in nodejs environments
+  cacheSensitivity: 0.01,
+  face: { enabled: true, detector: { maxDetected: 100, minConfidence: 0.1 } },
+  object: { enabled: true, maxDetected: 100, minConfidence: 0.1 },
   gesture: { enabled: true },
-  hand: { enabled: true },
-  body: { enabled: true, modelPath: 'https://vladmandic.github.io/human-models/models/movenet-multipose.json' },
+  hand: { enabled: true, maxDetected: 100, minConfidence: 0.2 },
+  body: { enabled: true, maxDetected: 100, minConfidence: 0.1, modelPath: 'https://vladmandic.github.io/human-models/models/movenet-multipose.json' },
 };
 
-async function main() {
-  log.header();
+const poolSize = 4;
 
-  globalThis.Canvas = canvas.Canvas; // patch global namespace with canvas library
-  globalThis.ImageData = canvas.ImageData; // patch global namespace with canvas library
+const human = new Human.Human(config); // create instance of human
 
-  const human = new Human.Human(config); // create instance of human
-  log.info('Human:', human.version, 'TF:', tf.version_core);
-  const configErrors = await human.validate();
-  if (configErrors.length > 0) log.error('Configuration errors:', configErrors);
-  await human.load(); // pre-load models
-  log.info('Loaded models:', Object.keys(human.models).filter((a) => human.models[a]));
+async function saveFile(shape, buffer, result, outFile) {
+  return new Promise(async (resolve, reject) => { // eslint-disable-line no-async-promise-executor
+    const outputCanvas = new canvas.Canvas(shape[2], shape[1]); // create canvas
+    const outputCtx = outputCanvas.getContext('2d');
+    const inputImage = await canvas.loadImage(buffer); // load image using canvas library
+    outputCtx.drawImage(inputImage, 0, 0); // draw input image onto canvas
+    human.draw.all(outputCanvas, result); // use human build-in method to draw results as overlays on canvas
+    const outStream = fs.createWriteStream(outFile); // write canvas to new image file
+    outStream.on('finish', () => {
+      log.data('Output image:', outFile, outputCanvas.width, outputCanvas.height);
+      resolve();
+    });
+    outStream.on('error', (err) => {
+      log.error('Output error:', outFile, err);
+      reject();
+    });
+    const stream = outputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true });
+    stream.pipe(outStream);
+  });
+}
 
-  const inDir = process.argv[2];
-  const outDir = process.argv[3];
-  if (process.argv.length !== 4) {
-    log.error('Parameters: <input-directory> <output-directory> missing');
-    return;
-  }
-  if (!fs.existsSync(inDir) || !fs.statSync(inDir).isDirectory() || !fs.existsSync(outDir) || !fs.statSync(outDir).isDirectory()) {
-    log.error('Invalid directory specified:', 'input:', fs.existsSync(inDir) ?? fs.statSync(inDir).isDirectory(), 'output:', fs.existsSync(outDir) ?? fs.statSync(outDir).isDirectory());
-    return;
-  }
-
-  const dir = fs.readdirSync(inDir);
-  const images = dir.filter((f) => fs.statSync(path.join(inDir, f)).isFile() && (f.toLocaleLowerCase().endsWith('.jpg') || f.toLocaleLowerCase().endsWith('.jpeg')));
-  log.info(`Processing folder: ${inDir} entries:`, dir.length, 'images', images.length);
-  for (const image of images) {
-    const inFile = path.join(inDir, image);
+async function processFile(image, inFile, outFile) {
   const buffer = fs.readFileSync(inFile);
-    const tensor = human.tf.tidy(() => {
-      const decode = human.tf.node.decodeImage(buffer, 3);
-      const expand = human.tf.expandDims(decode, 0);
-      const cast = human.tf.cast(expand, 'float32');
+  const tensor = tf.tidy(() => {
+    const decode = tf.node.decodeImage(buffer, 3);
+    const expand = tf.expandDims(decode, 0);
+    const cast = tf.cast(expand, 'float32');
     return cast;
   });
   log.state('Loaded image:', inFile, tensor.shape);
@@ -69,18 +67,54 @@ async function main() {
   human.tf.dispose(tensor);
   log.data(`Detected: ${image}:`, 'Face:', result.face.length, 'Body:', result.body.length, 'Hand:', result.hand.length, 'Objects:', result.object.length, 'Gestures:', result.gesture.length);
 
-    const outputCanvas = new canvas.Canvas(tensor.shape[2], tensor.shape[1]); // create canvas
-    const outputCtx = outputCanvas.getContext('2d');
-    const inputImage = await canvas.loadImage(buffer); // load image using canvas library
-    outputCtx.drawImage(inputImage, 0, 0); // draw input image onto canvas
-    human.draw.all(outputCanvas, result); // use human build-in method to draw results as overlays on canvas
-    const outFile = path.join(outDir, image);
-    const outStream = fs.createWriteStream(outFile); // write canvas to new image file
-    outStream.on('finish', () => log.state('Output image:', outFile, outputCanvas.width, outputCanvas.height));
-    outStream.on('error', (err) => log.error('Output error:', outFile, err));
-    const stream = outputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true });
-    stream.pipe(outStream);
+  if (outFile) await saveFile(tensor.shape, buffer, result, outFile);
+}
+
+async function main() {
+  log.header();
+
+  globalThis.Canvas = canvas.Canvas; // patch global namespace with canvas library
+  globalThis.ImageData = canvas.ImageData; // patch global namespace with canvas library
+
+  log.info('Human:', human.version, 'TF:', tf.version_core);
+  process.noDeprecation = true;
+  const configErrors = await human.validate();
+  if (configErrors.length > 0) log.error('Configuration errors:', configErrors);
+  await human.load(); // pre-load models
+  log.info('Loaded models:', human.models.loaded());
+
+  const inDir = process.argv[2];
+  const outDir = process.argv[3];
+  if (!inDir) {
+    log.error('Parameters: <input-directory> missing');
+    return;
   }
+  if (inDir && (!fs.existsSync(inDir) || !fs.statSync(inDir).isDirectory())) {
+    log.error('Invalid input directory:', fs.existsSync(inDir) ?? fs.statSync(inDir).isDirectory());
+    return;
+  }
+  if (!outDir) {
+    log.info('Parameters: <output-directory> missing, images will not be saved');
+  }
+  if (outDir && (!fs.existsSync(outDir) || !fs.statSync(outDir).isDirectory())) {
+    log.error('Invalid output directory:', fs.existsSync(outDir) ?? fs.statSync(outDir).isDirectory());
+    return;
+  }
+
+  const dir = fs.readdirSync(inDir);
+  const images = dir.filter((f) => fs.statSync(path.join(inDir, f)).isFile() && (f.toLocaleLowerCase().endsWith('.jpg') || f.toLocaleLowerCase().endsWith('.jpeg')));
+  log.info(`Processing folder: ${inDir} entries:`, dir.length, 'images', images.length);
+  const t0 = performance.now();
+  const promises = [];
+  for (let i = 0; i < images.length; i++) {
+    const inFile = path.join(inDir, images[i]);
+    const outFile = outDir ? path.join(outDir, images[i]) : null;
+    promises.push(processFile(images[i], inFile, outFile));
+    if (i % poolSize === 0) await Promise.all(promises);
+  }
+  await Promise.all(promises);
+  const t1 = performance.now();
+  log.info(`Processed ${images.length} images in ${Math.round(t1 - t0)} ms`);
 }
 
 main();

demo/segmentation/index.html

@@ -0,0 +1,61 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="content-type" content="text/html; charset=utf-8">
+    <title>Human Demo</title>
+    <meta name="viewport" content="width=device-width, shrink-to-fit=yes">
+    <meta name="mobile-web-app-capable" content="yes">
+    <meta name="application-name" content="Human Demo">
+    <meta name="keywords" content="Human Demo">
+    <meta name="description" content="Human Demo; Author: Vladimir Mandic <mandic00@live.com>">
+    <link rel="manifest" href="../manifest.webmanifest">
+    <link rel="shortcut icon" href="../favicon.ico" type="image/x-icon">
+    <link rel="icon" sizes="256x256" href="../assets/icons/dash-256.png">
+    <link rel="apple-touch-icon" href="../assets/icons/dash-256.png">
+    <link rel="apple-touch-startup-image" href="../assets/icons/dash-256.png">
+    <style>
+      @font-face { font-family: 'CenturyGothic'; font-display: swap; font-style: normal; font-weight: 400; src: local('CenturyGothic'), url('../assets/century-gothic.ttf') format('truetype'); }
+      html { font-size: 18px; }
+      body { font-size: 1rem; font-family: "CenturyGothic", "Segoe UI", sans-serif; font-variant: small-caps; width: -webkit-fill-available; height: 100%; background: black; color: white; overflow: hidden; margin: 0; }
+      select { font-size: 1rem; font-family: "CenturyGothic", "Segoe UI", sans-serif; font-variant: small-caps; background: gray; color: white; border: none; }
+    </style>
+    <script src="../segmentation/index.js" type="module"></script>
+  </head>
+  <body>
+    <noscript><h1>javascript is required</h1></noscript>
+    <nav>
+      <div id="nav" class="nav"></div>
+    </nav>
+    <header>
+      <div id="header" class="header" style="position: fixed; top: 0; right: 0; padding: 4px; margin: 16px; background: rgba(0, 0, 0, 0.5); z-index: 10; line-height: 2rem;">
+        <label for="mode">mode</label>
+        <select id="mode" name="mode">
+          <option value="default">remove background</option>
+          <option value="alpha">draw alpha channel</option>
+          <option value="foreground">full foreground</option>
+          <option value="state">recurrent state</option>
+        </select><br>
+        <label for="composite">composite</label>
+        <select id="composite" name="composite"></select><br>
+        <label for="ratio">downsample ratio</label>
+        <input type="range" name="ratio" id="ratio" min="0.1" max="1" value="0.5" step="0.05">
+        <div id="fps" style="margin-top: 8px"></div>
+      </div>
+    </header>
+    <main>
+      <div id="main" class="main">
+        <video id="webcam" style="position: fixed; top: 0; left: 0; width: 50vw; height: 50vh"></video>
+        <img id="background" alt="background" style="position: fixed; top: 0; right: 0; width: 50vw; height: 50vh" controls></img>
+        <canvas id="output" style="position: fixed; bottom: 0; left: 0; height: 50vh"></canvas>
+        <canvas id="merge" style="position: fixed; bottom: 0; right: 0; height: 50vh"></canvas>
+      </div>
+    </main>
+    <footer>
+      <div id="footer" class="footer"></div>
+    </footer>
+    <aside>
+      <div id="aside" class="aside"></div>
+    </aside>
+  </body>
+</html>

demo/segmentation/index.js

@@ -0,0 +1,99 @@
+/**
+ * Human demo for browsers
+ * @default Human Library
+ * @summary <https://github.com/vladmandic/human>
+ * @author <https://github.com/vladmandic>
+ * @copyright <https://github.com/vladmandic>
+ * @license MIT
+ */
+
+import * as H from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human
+
+const humanConfig = { // user configuration for human, used to fine-tune behavior
+  modelBasePath: 'https://vladmandic.github.io/human-models/models/',
+  filter: { enabled: true, equalization: false, flip: false },
+  face: { enabled: false },
+  body: { enabled: false },
+  hand: { enabled: false },
+  object: { enabled: false },
+  gesture: { enabled: false },
+  segmentation: {
+    enabled: true,
+    modelPath: 'rvm.json', // can use rvm, selfie or meet
+    ratio: 0.5,
+    mode: 'default',
+  },
+};
+
+const backgroundImage = '../../samples/in/background.jpg';
+
+const human = new H.Human(humanConfig); // create instance of human with overrides from user configuration
+
+const log = (...msg) => console.log(...msg); // eslint-disable-line no-console
+
+async function main() {
+  // gather dom elements
+  const dom = {
+    background: document.getElementById('background'),
+    webcam: document.getElementById('webcam'),
+    output: document.getElementById('output'),
+    merge: document.getElementById('merge'),
+    mode: document.getElementById('mode'),
+    composite: document.getElementById('composite'),
+    ratio: document.getElementById('ratio'),
+    fps: document.getElementById('fps'),
+  };
+  // set defaults
+  dom.fps.innerText = 'initializing';
+  dom.ratio.valueAsNumber = human.config.segmentation.ratio;
+  dom.background.src = backgroundImage;
+  dom.composite.innerHTML = ['source-atop', 'color', 'color-burn', 'color-dodge', 'copy', 'darken', 'destination-atop', 'destination-in', 'destination-out', 'destination-over', 'difference', 'exclusion', 'hard-light', 'hue', 'lighten', 'lighter', 'luminosity', 'multiply', 'overlay', 'saturation', 'screen', 'soft-light', 'source-in', 'source-out', 'source-over', 'xor'].map((gco) => `<option value="${gco}">${gco}</option>`).join(''); // eslint-disable-line max-len
+  const ctxMerge = dom.merge.getContext('2d');
+
+  log('human version:', human.version, '| tfjs version:', human.tf.version['tfjs-core']);
+  log('platform:', human.env.platform, '| agent:', human.env.agent);
+  await human.load(); // preload all models
+  log('backend:', human.tf.getBackend(), '| available:', human.env.backends);
+  log('models stats:', human.models.stats());
+  log('models loaded:', human.models.loaded());
+  await human.warmup(); // warmup function to initialize backend for future faster detection
+  const numTensors = human.tf.engine().state.numTensors;
+
+  // initialize webcam
+  dom.webcam.onplay = () => { // start processing on video play
+    log('start processing');
+    dom.output.width = human.webcam.width;
+    dom.output.height = human.webcam.height;
+    dom.merge.width = human.webcam.width;
+    dom.merge.height = human.webcam.height;
+    loop(); // eslint-disable-line no-use-before-define
+  };
+
+  await human.webcam.start({ element: dom.webcam, crop: true, width: window.innerWidth / 2, height: window.innerHeight / 2 }); // use human webcam helper methods and associate webcam stream with a dom element
+  if (!human.webcam.track) dom.fps.innerText = 'webcam error';
+
+  // processing loop
+  async function loop() {
+    if (!human.webcam.element || human.webcam.paused) return; // check if webcam is valid and playing
+    human.config.segmentation.mode = dom.mode.value; // get segmentation mode from ui
+    human.config.segmentation.ratio = dom.ratio.valueAsNumber; // get segmentation downsample ratio from ui
+    const t0 = Date.now();
+    const rgba = await human.segmentation(human.webcam.element, human.config); // run model and process results
+    const t1 = Date.now();
+    if (!rgba) {
+      dom.fps.innerText = 'error';
+      return;
+    }
+    dom.fps.innerText = `fps: ${Math.round(10000 / (t1 - t0)) / 10}`; // mark performance
+    human.draw.tensor(rgba, dom.output); // draw raw output
+    human.tf.dispose(rgba); // dispose tensors
+    ctxMerge.globalCompositeOperation = 'source-over';
+    ctxMerge.drawImage(dom.background, 0, 0); // draw original video to first stacked canvas
+    ctxMerge.globalCompositeOperation = dom.composite.value;
+    ctxMerge.drawImage(dom.output, 0, 0); // draw processed output to second stacked canvas
+    if (numTensors !== human.tf.engine().state.numTensors) log({ leak: human.tf.engine().state.numTensors - numTensors }); // check for memory leaks
+    requestAnimationFrame(loop);
+  }
+}
+
+window.onload = main;

demo/tracker/README.md

@@ -0,0 +1,28 @@
+## Tracker
+
+### Based on
+
+<https://github.com/opendatacam/node-moving-things-tracker>  
+
+### Build
+
+- remove reference to `lodash`:  
+  > `isEqual` in <tracker.js>  
+- replace external lib:  
+  > curl https://raw.githubusercontent.com/ubilabs/kd-tree-javascript/master/kdTree.js -o lib/kdTree-min.js
+- build with `esbuild`:  
+  > node_modules/.bin/esbuild --bundle tracker.js --format=esm --platform=browser --target=esnext --keep-names --tree-shaking=false --analyze --outfile=/home/vlado/dev/human/demo/tracker/tracker.js --banner:js="/* eslint-disable */"
+
+### Usage
+
+    computeDistance(item1, item2)
+    disableKeepInMemory()
+    enableKeepInMemory()
+    getAllTrackedItems()
+    getJSONDebugOfTrackedItems(roundInt = true)
+    getJSONOfAllTrackedItems()
+    getJSONOfTrackedItems(roundInt = true)
+    getTrackedItemsInMOTFormat(frameNb)
+    reset()
+    setParams(newParams)
+    updateTrackedItemsWithNewFrame(detectionsOfThisFrame, frameNb)

demo/tracker/index.html

@@ -0,0 +1,65 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Human</title>
+    <meta name="viewport" content="width=device-width" id="viewport">
+    <meta name="keywords" content="Human">
+    <meta name="application-name" content="Human">
+    <meta name="description" content="Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition; Author: Vladimir Mandic <https://github.com/vladmandic>">
+    <meta name="msapplication-tooltip" content="Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition; Author: Vladimir Mandic <https://github.com/vladmandic>">
+    <meta name="theme-color" content="#000000">
+    <link rel="manifest" href="../manifest.webmanifest">
+    <link rel="shortcut icon" href="../../favicon.ico" type="image/x-icon">
+    <link rel="apple-touch-icon" href="../../assets/icon.png">
+    <script src="./index.js" type="module"></script>
+    <style>
+      html { font-family: 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
+      body { margin: 0; background: black; color: white; overflow-x: hidden; width: 100vw; height: 100vh; }
+      body::-webkit-scrollbar { display: none; }
+      input[type="file"] { font-family: 'Segoe UI'; font-size: 14px; font-variant: small-caps; }
+      ::-webkit-file-upload-button { background: #333333; color: white; border: 0; border-radius: 0; padding: 6px 16px; box-shadow: 4px 4px 4px #222222; font-family: 'Segoe UI'; font-size: 14px; font-variant: small-caps; }
+    </style>
+  </head>
+  <body>
+    <div style="display: flex">
+      <video id="video" playsinline style="width: 25vw" controls controlslist="nofullscreen nodownload noremoteplayback" disablepictureinpicture loop></video>
+      <canvas id="canvas" style="width: 75vw"></canvas> 
+    </div>
+    <div class="uploader" style="padding: 8px">
+      <input type="file" name="inputvideo" id="inputvideo" accept="video/*"></input>
+      <input type="checkbox" id="interpolation" name="interpolation"></input>
+      <label for="tracker">interpolation</label>
+    </div>
+    <form id="config" style="padding: 8px; line-height: 1.6rem;">
+      tracker | 
+      <input type="checkbox" id="tracker" name="tracker" checked></input>
+      <label for="tracker">enabled</label> | 
+      <input type="checkbox" id="keepInMemory" name="keepInMemory"></input>
+      <label for="keepInMemory">keepInMemory</label> | 
+      <br>
+      tracker source | 
+      <input type="radio" id="box-face" name="box" value="face" checked>
+      <label for="box-face">face</label> |
+      <input type="radio" id="box-body" name="box" value="body">
+      <label for="box-face">body</label> |
+      <input type="radio" id="box-object" name="box" value="object">
+      <label for="box-face">object</label> |
+      <br>
+      tracker config | 
+      <input type="range" id="unMatchedFramesTolerance" name="unMatchedFramesTolerance" min="0" max="300" step="1", value="60"></input>
+      <label for="unMatchedFramesTolerance">unMatchedFramesTolerance</label> |
+      <input type="range" id="iouLimit" name="unMatchedFramesTolerance" min="0" max="1" step="0.01", value="0.1"></input>
+      <label for="iouLimit">iouLimit</label> |
+      <input type="range" id="distanceLimit" name="unMatchedFramesTolerance" min="0" max="1" step="0.01", value="0.1"></input>
+      <label for="distanceLimit">distanceLimit</label> |
+      <input type="radio" id="matchingAlgorithm-kdTree" name="matchingAlgorithm" value="kdTree" checked>
+      <label for="matchingAlgorithm-kdTree">kdTree</label> |
+      <input type="radio" id="matchingAlgorithm-munkres" name="matchingAlgorithm" value="munkres">
+      <label for="matchingAlgorithm-kdTree">munkres</label> |
+    </form>
+    <pre id="status" style="position: absolute; top: 12px; right: 20px; background-color: grey; padding: 8px; box-shadow: 2px 2px black"></pre>
+    <pre id="log" style="padding: 8px"></pre>
+    <div id="performance" style="position: absolute; bottom: 0; width: 100%; padding: 8px; font-size: 0.8rem;"></div>
+  </body>
+</html>

demo/tracker/index.ts

@@ -0,0 +1,208 @@
+/**
+ * Human demo for browsers
+ * @default Human Library
+ * @summary <https://github.com/vladmandic/human>
+ * @author <https://github.com/vladmandic>
+ * @copyright <https://github.com/vladmandic>
+ * @license MIT
+ */
+
+import * as H from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human
+import tracker from './tracker.js';
+
+const humanConfig: Partial<H.Config> = { // user configuration for human, used to fine-tune behavior
+  debug: true,
+  backend: 'webgl',
+  // cacheSensitivity: 0,
+  // cacheModels: false,
+  // warmup: 'none',
+  modelBasePath: 'https://vladmandic.github.io/human-models/models',
+  filter: { enabled: true, equalization: false, flip: false },
+  face: {
+    enabled: true,
+    detector: { rotation: false, maxDetected: 10, minConfidence: 0.3 },
+    mesh: { enabled: true },
+    attention: { enabled: false },
+    iris: { enabled: false },
+    description: { enabled: false },
+    emotion: { enabled: false },
+    antispoof: { enabled: false },
+    liveness: { enabled: false },
+  },
+  body: { enabled: false, maxDetected: 6, modelPath: 'movenet-multipose.json' },
+  hand: { enabled: false },
+  object: { enabled: false, maxDetected: 10 },
+  segmentation: { enabled: false },
+  gesture: { enabled: false },
+};
+
+interface TrackerConfig {
+  unMatchedFramesTolerance: number, // number of frame when an object is not matched before considering it gone; ignored if fastDelete is set
+  iouLimit: number, // exclude things from beeing matched if their IOU less than; 1 means total overlap; 0 means no overlap
+  fastDelete: boolean, // remove new objects immediately if they could not be matched in the next frames; if set, ignores unMatchedFramesTolerance
+  distanceLimit: number, // distance limit for matching; if values need to be excluded from matching set their distance to something greater than the distance limit
+  matchingAlgorithm: 'kdTree' | 'munkres', // algorithm used to match tracks with new detections
+}
+
+interface TrackerResult {
+  id: number,
+  confidence: number,
+  bearing: number,
+  isZombie: boolean,
+  name: string,
+  x: number,
+  y: number,
+  w: number,
+  h: number,
+}
+
+const trackerConfig: TrackerConfig = {
+  unMatchedFramesTolerance: 100,
+  iouLimit: 0.05,
+  fastDelete: false,
+  distanceLimit: 1e4,
+  matchingAlgorithm: 'kdTree',
+};
+
+const human = new H.Human(humanConfig); // create instance of human with overrides from user configuration
+
+const dom = { // grab instances of dom objects so we dont have to look them up later
+  video: document.getElementById('video') as HTMLVideoElement,
+  canvas: document.getElementById('canvas') as HTMLCanvasElement,
+  log: document.getElementById('log') as HTMLPreElement,
+  fps: document.getElementById('status') as HTMLPreElement,
+  tracker: document.getElementById('tracker') as HTMLInputElement,
+  interpolation: document.getElementById('interpolation') as HTMLInputElement,
+  config: document.getElementById('config') as HTMLFormElement,
+  ctx: (document.getElementById('canvas') as HTMLCanvasElement).getContext('2d') as CanvasRenderingContext2D,
+};
+const timestamp = { detect: 0, draw: 0, tensors: 0, start: 0 }; // holds information used to calculate performance and possible memory leaks
+const fps = { detectFPS: 0, drawFPS: 0, frames: 0, averageMs: 0 }; // holds calculated fps information for both detect and screen refresh
+
+const log = (...msg) => { // helper method to output messages
+  dom.log.innerText += msg.join(' ') + '\n';
+  console.log(...msg); // eslint-disable-line no-console
+};
+const status = (msg) => dom.fps.innerText = msg; // print status element
+
+async function detectionLoop() { // main detection loop
+  if (!dom.video.paused && dom.video.readyState >= 2) {
+    if (timestamp.start === 0) timestamp.start = human.now();
+    // log('profiling data:', await human.profile(dom.video));
+    await human.detect(dom.video, humanConfig); // actual detection; were not capturing output in a local variable as it can also be reached via human.result
+    const tensors = human.tf.memory().numTensors; // check current tensor usage for memory leaks
+    if (tensors - timestamp.tensors !== 0) log('allocated tensors:', tensors - timestamp.tensors); // printed on start and each time there is a tensor leak
+    timestamp.tensors = tensors;
+    fps.detectFPS = Math.round(1000 * 1000 / (human.now() - timestamp.detect)) / 1000;
+    fps.frames++;
+    fps.averageMs = Math.round(1000 * (human.now() - timestamp.start) / fps.frames) / 1000;
+  }
+  timestamp.detect = human.now();
+  requestAnimationFrame(detectionLoop); // start new frame immediately
+}
+
+function drawLoop() { // main screen refresh loop
+  if (!dom.video.paused && dom.video.readyState >= 2) {
+    const res: H.Result = dom.interpolation.checked ? human.next(human.result) : human.result; // interpolate results if enabled
+    let tracking: H.FaceResult[] | H.BodyResult[] | H.ObjectResult[] = [];
+    if (human.config.face.enabled) tracking = res.face;
+    else if (human.config.body.enabled) tracking = res.body;
+    else if (human.config.object.enabled) tracking = res.object;
+    else log('unknown object type');
+    let data: TrackerResult[] = [];
+    if (dom.tracker.checked) {
+      const items = tracking.map((obj) => ({
+        x: obj.box[0] + obj.box[2] / 2,
+        y: obj.box[1] + obj.box[3] / 2,
+        w: obj.box[2],
+        h: obj.box[3],
+        name: obj.label || (human.config.face.enabled ? 'face' : 'body'),
+        confidence: obj.score,
+      }));
+      tracker.updateTrackedItemsWithNewFrame(items, fps.frames);
+      data = tracker.getJSONOfTrackedItems(true) as TrackerResult[];
+    }
+    human.draw.canvas(dom.video, dom.canvas); // copy input video frame to output canvas
+    for (let i = 0; i < tracking.length; i++) {
+      // @ts-ignore
+      const name = tracking[i].label || (human.config.face.enabled ? 'face' : 'body');
+      dom.ctx.strokeRect(tracking[i].box[0], tracking[i].box[1], tracking[i].box[1], tracking[i].box[2]);
+      dom.ctx.fillText(`id: ${tracking[i].id} ${Math.round(100 * tracking[i].score)}% ${name}`, tracking[i].box[0] + 4, tracking[i].box[1] + 16);
+      if (data[i]) {
+        dom.ctx.fillText(`t: ${data[i].id} ${Math.round(100 * data[i].confidence)}% ${data[i].name} ${data[i].isZombie ? 'zombie' : ''}`, tracking[i].box[0] + 4, tracking[i].box[1] + 34);
+      }
+    }
+  }
+  const now = human.now();
+  fps.drawFPS = Math.round(1000 * 1000 / (now - timestamp.draw)) / 1000;
+  timestamp.draw = now;
+  status(dom.video.paused ? 'paused' : `fps: ${fps.detectFPS.toFixed(1).padStart(5, ' ')} detect | ${fps.drawFPS.toFixed(1).padStart(5, ' ')} draw`); // write status
+  setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 30 fps
+}
+
+async function handleVideo(file: File) {
+  const url = URL.createObjectURL(file);
+  dom.video.src = url;
+  await dom.video.play();
+  log('loaded video:', file.name, 'resolution:', [dom.video.videoWidth, dom.video.videoHeight], 'duration:', dom.video.duration);
+  dom.canvas.width = dom.video.videoWidth;
+  dom.canvas.height = dom.video.videoHeight;
+  dom.ctx.strokeStyle = 'white';
+  dom.ctx.fillStyle = 'white';
+  dom.ctx.font = '16px Segoe UI';
+  dom.video.playbackRate = 0.25;
+}
+
+function initInput() {
+  document.body.addEventListener('dragenter', (evt) => evt.preventDefault());
+  document.body.addEventListener('dragleave', (evt) => evt.preventDefault());
+  document.body.addEventListener('dragover', (evt) => evt.preventDefault());
+  document.body.addEventListener('drop', async (evt) => {
+    evt.preventDefault();
+    if (evt.dataTransfer) evt.dataTransfer.dropEffect = 'copy';
+    const file = evt.dataTransfer?.files?.[0];
+    if (file) await handleVideo(file);
+    log(dom.video.readyState);
+  });
+  (document.getElementById('inputvideo') as HTMLInputElement).onchange = async (evt) => {
+    evt.preventDefault();
+    const file = evt.target?.['files']?.[0];
+    if (file) await handleVideo(file);
+  };
+  dom.config.onchange = () => {
+    trackerConfig.distanceLimit = (document.getElementById('distanceLimit') as HTMLInputElement).valueAsNumber;
+    trackerConfig.iouLimit = (document.getElementById('iouLimit') as HTMLInputElement).valueAsNumber;
+    trackerConfig.unMatchedFramesTolerance = (document.getElementById('unMatchedFramesTolerance') as HTMLInputElement).valueAsNumber;
+    trackerConfig.unMatchedFramesTolerance = (document.getElementById('unMatchedFramesTolerance') as HTMLInputElement).valueAsNumber;
+    trackerConfig.matchingAlgorithm = (document.getElementById('matchingAlgorithm-kdTree') as HTMLInputElement).checked ? 'kdTree' : 'munkres';
+    tracker.setParams(trackerConfig);
+    if ((document.getElementById('keepInMemory') as HTMLInputElement).checked) tracker.enableKeepInMemory();
+    else tracker.disableKeepInMemory();
+    tracker.reset();
+    log('tracker config change', JSON.stringify(trackerConfig));
+    humanConfig.face!.enabled = (document.getElementById('box-face') as HTMLInputElement).checked; // eslint-disable-line @typescript-eslint/no-non-null-assertion
+    humanConfig.body!.enabled = (document.getElementById('box-body') as HTMLInputElement).checked; // eslint-disable-line @typescript-eslint/no-non-null-assertion
+    humanConfig.object!.enabled = (document.getElementById('box-object') as HTMLInputElement).checked; // eslint-disable-line @typescript-eslint/no-non-null-assertion
+  };
+  dom.tracker.onchange = (evt) => {
+    log('tracker', (evt.target as HTMLInputElement).checked ? 'enabled' : 'disabled');
+    tracker.setParams(trackerConfig);
+    tracker.reset();
+  };
+}
+
+async function main() { // main entry point
+  log('human version:', human.version, '| tfjs version:', human.tf.version['tfjs-core']);
+  log('platform:', human.env.platform, '| agent:', human.env.agent);
+  status('loading...');
+  await human.load(); // preload all models
+  log('backend:', human.tf.getBackend(), '| available:', human.env.backends);
+  log('models loaded:', human.models.loaded());
+  status('initializing...');
+  await human.warmup(); // warmup function to initialize backend for future faster detection
+  initInput(); // initialize input
+  await detectionLoop(); // start detection loop
+  drawLoop(); // start draw loop
+}
+
+window.onload = main;

demo/typescript/index.html

@@ -21,7 +21,7 @@
     </style>
   </head>
   <body>
-    <canvas id="canvas" style="margin: 0 auto; width: 100%"></canvas>
+    <canvas id="canvas" style="margin: 0 auto; width: 100vw"></canvas>
     <video id="video" playsinline style="display: none"></video>
     <pre id="status" style="position: absolute; top: 12px; right: 20px; background-color: grey; padding: 8px; box-shadow: 2px 2px black"></pre>
     <pre id="log" style="padding: 8px"></pre>

demo/typescript/index.js

@@ -4,6 +4,6 @@
   author: <https://github.com/vladmandic>'
 */
 
-import*as c from"../../dist/human.esm.js";var w={async:!1,modelBasePath:"../../models",filter:{enabled:!0,equalization:!1,flip:!1},face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},attention:{enabled:!1},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!1},gesture:{enabled:!0}},e=new c.Human(w);e.env.perfadd=!1;e.draw.options.font='small-caps 18px "Lato"';e.draw.options.lineHeight=20;var t={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},n={detect:0,draw:0,tensors:0,start:0},o={detectFPS:0,drawFPS:0,frames:0,averageMs:0},i=(...a)=>{t.log.innerText+=a.join(" ")+`
-`,console.log(...a)},r=a=>t.fps.innerText=a,b=a=>t.perf.innerText="tensors:"+e.tf.memory().numTensors.toString()+" | performance: "+JSON.stringify(a).replace(/"|{|}/g,"").replace(/,/g," | ");async function h(){r("starting webcam...");let a={audio:!1,video:{facingMode:"user",resizeMode:"none",width:{ideal:document.body.clientWidth},height:{ideal:document.body.clientHeight}}},d=await navigator.mediaDevices.getUserMedia(a),f=new Promise(p=>{t.video.onloadeddata=()=>p(!0)});t.video.srcObject=d,t.video.play(),await f,t.canvas.width=t.video.videoWidth,t.canvas.height=t.video.videoHeight;let s=d.getVideoTracks()[0],v=s.getCapabilities?s.getCapabilities():"",g=s.getSettings?s.getSettings():"",u=s.getConstraints?s.getConstraints():"";i("video:",t.video.videoWidth,t.video.videoHeight,s.label,{stream:d,track:s,settings:g,constraints:u,capabilities:v}),t.canvas.onclick=()=>{t.video.paused?t.video.play():t.video.pause()}}async function l(){if(!t.video.paused){n.start===0&&(n.start=e.now()),await e.detect(t.video);let a=e.tf.memory().numTensors;a-n.tensors!==0&&i("allocated tensors:",a-n.tensors),n.tensors=a,o.detectFPS=Math.round(1e3*1e3/(e.now()-n.detect))/1e3,o.frames++,o.averageMs=Math.round(1e3*(e.now()-n.start)/o.frames)/1e3,o.frames%100===0&&!t.video.paused&&i("performance",{...o,tensors:n.tensors})}n.detect=e.now(),requestAnimationFrame(l)}async function m(){if(!t.video.paused){let d=e.next(e.result);e.config.filter.flip?e.draw.canvas(d.canvas,t.canvas):e.draw.canvas(t.video,t.canvas),await e.draw.all(t.canvas,d),b(d.performance)}let a=e.now();o.drawFPS=Math.round(1e3*1e3/(a-n.draw))/1e3,n.draw=a,r(t.video.paused?"paused":`fps: ${o.detectFPS.toFixed(1).padStart(5," ")} detect | ${o.drawFPS.toFixed(1).padStart(5," ")} draw`),setTimeout(m,30)}async function M(){i("human version:",e.version,"| tfjs version:",e.tf.version["tfjs-core"]),i("platform:",e.env.platform,"| agent:",e.env.agent),r("loading..."),await e.load(),i("backend:",e.tf.getBackend(),"| available:",e.env.backends),i("models stats:",e.getModelStats()),i("models loaded:",Object.values(e.models).filter(a=>a!==null).length),r("initializing..."),await e.warmup(),await h(),await l(),await m()}window.onload=M;
+import*as m from"../../dist/human.esm.js";var v=1920,b={debug:!0,backend:"webgl",modelBasePath:"https://vladmandic.github.io/human-models/models/",filter:{enabled:!0,equalization:!1,flip:!1},face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},attention:{enabled:!1},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0},antispoof:{enabled:!0},liveness:{enabled:!0}},body:{enabled:!1},hand:{enabled:!1},object:{enabled:!1},segmentation:{enabled:!1},gesture:{enabled:!0}},e=new m.Human(b);e.env.perfadd=!1;e.draw.options.font='small-caps 18px "Lato"';e.draw.options.lineHeight=20;e.draw.options.drawPoints=!0;var a={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},n={detect:0,draw:0,tensors:0,start:0},s={detectFPS:0,drawFPS:0,frames:0,averageMs:0},o=(...t)=>{a.log.innerText+=t.join(" ")+`
+`,console.log(...t)},i=t=>a.fps.innerText=t,g=t=>a.perf.innerText="tensors:"+e.tf.memory().numTensors.toString()+" | performance: "+JSON.stringify(t).replace(/"|{|}/g,"").replace(/,/g," | ");async function f(){if(!a.video.paused){n.start===0&&(n.start=e.now()),await e.detect(a.video);let t=e.tf.memory().numTensors;t-n.tensors!==0&&o("allocated tensors:",t-n.tensors),n.tensors=t,s.detectFPS=Math.round(1e3*1e3/(e.now()-n.detect))/1e3,s.frames++,s.averageMs=Math.round(1e3*(e.now()-n.start)/s.frames)/1e3,s.frames%100===0&&!a.video.paused&&o("performance",{...s,tensors:n.tensors})}n.detect=e.now(),requestAnimationFrame(f)}async function u(){var d,r,c;if(!a.video.paused){let l=e.next(e.result),w=await e.image(a.video);e.draw.canvas(w.canvas,a.canvas);let p={bodyLabels:`person confidence [score] and ${(c=(r=(d=e.result)==null?void 0:d.body)==null?void 0:r[0])==null?void 0:c.keypoints.length} keypoints`};await e.draw.all(a.canvas,l,p),g(l.performance)}let t=e.now();s.drawFPS=Math.round(1e3*1e3/(t-n.draw))/1e3,n.draw=t,i(a.video.paused?"paused":`fps: ${s.detectFPS.toFixed(1).padStart(5," ")} detect | ${s.drawFPS.toFixed(1).padStart(5," ")} draw`),setTimeout(u,30)}async function h(){let d=(await e.webcam.enumerate())[0].deviceId,r=await e.webcam.start({element:a.video,crop:!1,width:v,id:d});o(r),a.canvas.width=e.webcam.width,a.canvas.height=e.webcam.height,a.canvas.onclick=async()=>{e.webcam.paused?await e.webcam.play():e.webcam.pause()}}async function y(){o("human version:",e.version,"| tfjs version:",e.tf.version["tfjs-core"]),o("platform:",e.env.platform,"| agent:",e.env.agent),i("loading..."),await e.load(),o("backend:",e.tf.getBackend(),"| available:",e.env.backends),o("models stats:",e.models.stats()),o("models loaded:",e.models.loaded()),o("environment",e.env),i("initializing..."),await e.warmup(),await h(),await f(),await u()}window.onload=y;
 //# sourceMappingURL=index.js.map

demo/typescript/index.ts

@@ -9,17 +9,22 @@
 
 import * as H from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human
 
+const width = 1920; // used by webcam config as well as human maximum resultion // can be anything, but resolutions higher than 4k will disable internal optimizations
+
 const humanConfig: Partial<H.Config> = { // user configuration for human, used to fine-tune behavior
-  // backend: 'wasm' as const,
-  // wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.20.0/dist/',
+  debug: true,
+  backend: 'webgl',
   // cacheSensitivity: 0,
-  async: false,
-  modelBasePath: '../../models',
+  // cacheModels: false,
+  // warmup: 'none',
+  // modelBasePath: '../../models',
+  modelBasePath: 'https://vladmandic.github.io/human-models/models/',
   filter: { enabled: true, equalization: false, flip: false },
-  face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } },
-  body: { enabled: true },
-  hand: { enabled: true },
+  face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true }, antispoof: { enabled: true }, liveness: { enabled: true } },
+  body: { enabled: false },
+  hand: { enabled: false },
   object: { enabled: false },
+  segmentation: { enabled: false },
   gesture: { enabled: true },
 };
 
@@ -28,6 +33,7 @@ const human = new H.Human(humanConfig); // create instance of human with overrid
 human.env.perfadd = false; // is performance data showing instant or total values
 human.draw.options.font = 'small-caps 18px "Lato"'; // set font used to draw labels when using draw methods
 human.draw.options.lineHeight = 20;
+human.draw.options.drawPoints = true; // draw points on face mesh
 // human.draw.options.fillPolygons = true;
 
 const dom = { // grab instances of dom objects so we dont have to look them up later
@@ -45,29 +51,7 @@ const log = (...msg) => { // helper method to output messages
   console.log(...msg); // eslint-disable-line no-console
 };
 const status = (msg) => dom.fps.innerText = msg; // print status element
-const perf = (msg) => dom.perf.innerText = 'tensors:' + (human.tf.memory().numTensors as number).toString() + ' | performance: ' + JSON.stringify(msg).replace(/"|{|}/g, '').replace(/,/g, ' | '); // print performance element
-
-async function webCam() { // initialize webcam
-  status('starting webcam...');
-  // @ts-ignore resizeMode is not yet defined in tslib
-  const options: MediaStreamConstraints = { audio: false, video: { facingMode: 'user', resizeMode: 'none', width: { ideal: document.body.clientWidth }, height: { ideal: document.body.clientHeight } } };
-  const stream: MediaStream = await navigator.mediaDevices.getUserMedia(options);
-  const ready = new Promise((resolve) => { dom.video.onloadeddata = () => resolve(true); });
-  dom.video.srcObject = stream;
-  void dom.video.play();
-  await ready;
-  dom.canvas.width = dom.video.videoWidth;
-  dom.canvas.height = dom.video.videoHeight;
-  const track: MediaStreamTrack = stream.getVideoTracks()[0];
-  const capabilities: MediaTrackCapabilities | string = track.getCapabilities ? track.getCapabilities() : '';
-  const settings: MediaTrackSettings | string = track.getSettings ? track.getSettings() : '';
-  const constraints: MediaTrackConstraints | string = track.getConstraints ? track.getConstraints() : '';
-  log('video:', dom.video.videoWidth, dom.video.videoHeight, track.label, { stream, track, settings, constraints, capabilities });
-  dom.canvas.onclick = () => { // pause when clicked on screen and resume on next click
-    if (dom.video.paused) void dom.video.play();
-    else dom.video.pause();
-  };
-}
+const perf = (msg) => dom.perf.innerText = 'tensors:' + human.tf.memory().numTensors.toString() + ' | performance: ' + JSON.stringify(msg).replace(/"|{|}/g, '').replace(/,/g, ' | '); // print performance element
 
 async function detectionLoop() { // main detection loop
   if (!dom.video.paused) {
@@ -89,9 +73,11 @@ async function detectionLoop() { // main detection loop
 async function drawLoop() { // main screen refresh loop
   if (!dom.video.paused) {
     const interpolated = human.next(human.result); // smoothen result using last-known results
-    if (human.config.filter.flip) human.draw.canvas(interpolated.canvas as HTMLCanvasElement, dom.canvas); // draw processed image to screen canvas
-    else human.draw.canvas(dom.video, dom.canvas); // draw original video to screen canvas // better than using procesed image as this loop happens faster than processing loop
-    await human.draw.all(dom.canvas, interpolated); // draw labels, boxes, lines, etc.
+    const processed = await human.image(dom.video); // get current video frame, but enhanced with human.filters
+    human.draw.canvas(processed.canvas as HTMLCanvasElement, dom.canvas);
+
+    const opt: Partial<H.DrawOptions> = { bodyLabels: `person confidence [score] and ${human.result?.body?.[0]?.keypoints.length} keypoints` };
+    await human.draw.all(dom.canvas, interpolated, opt); // draw labels, boxes, lines, etc.
     perf(interpolated.performance); // write performance data
   }
   const now = human.now();
@@ -101,14 +87,28 @@ async function drawLoop() { // main screen refresh loop
   setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 30 fps
 }
 
+async function webCam() {
+  const devices = await human.webcam.enumerate();
+  const id = devices[0].deviceId; // use first available video source
+  const webcamStatus = await human.webcam.start({ element: dom.video, crop: false, width, id }); // use human webcam helper methods and associate webcam stream with a dom element
+  log(webcamStatus);
+  dom.canvas.width = human.webcam.width;
+  dom.canvas.height = human.webcam.height;
+  dom.canvas.onclick = async () => { // pause when clicked on screen and resume on next click
+    if (human.webcam.paused) await human.webcam.play();
+    else human.webcam.pause();
+  };
+}
+
 async function main() { // main entry point
   log('human version:', human.version, '| tfjs version:', human.tf.version['tfjs-core']);
   log('platform:', human.env.platform, '| agent:', human.env.agent);
   status('loading...');
   await human.load(); // preload all models
   log('backend:', human.tf.getBackend(), '| available:', human.env.backends);
-  log('models stats:', human.getModelStats());
-  log('models loaded:', Object.values(human.models).filter((model) => model !== null).length);
+  log('models stats:', human.models.stats());
+  log('models loaded:', human.models.loaded());
+  log('environment', human.env);
   status('initializing...');
   await human.warmup(); // warmup function to initialize backend for future faster detection
   await webCam(); // start webcam

demo/video/index.html

@@ -0,0 +1,58 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Human</title>
+    <meta name="viewport" content="width=device-width" id="viewport">
+    <meta name="keywords" content="Human">
+    <meta name="description" content="Human: Demo; Author: Vladimir Mandic <https://github.com/vladmandic>">
+    <link rel="manifest" href="../manifest.webmanifest">
+    <link rel="shortcut icon" href="../../favicon.ico" type="image/x-icon">
+    <style>
+      @font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../../assets/lato-light.woff2') }
+      body { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; margin: 0; background: black; color: white; overflow: hidden; width: 100vw; height: 100vh; }
+    </style>
+  </head>
+  <body>
+    <canvas id="canvas" style="margin: 0 auto; width: 100%"></canvas>
+    <pre id="log" style="padding: 8px; position: fixed; bottom: 0"></pre>
+    <script type="module">
+      import * as H from '../../dist/human.esm.js'; // equivalent of import @vladmandic/Human
+
+      const humanConfig = { // user configuration for human, used to fine-tune behavior
+        modelBasePath: '../../models', // models can be loaded directly from cdn as well
+        filter: { enabled: true, equalization: true, flip: false },
+        face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } },
+        body: { enabled: true },
+        hand: { enabled: true },
+        gesture: { enabled: true },
+        object: { enabled: false },
+        segmentation: { enabled: false },
+      };
+      const human = new H.Human(humanConfig); // create instance of human with overrides from user configuration
+      const canvas = document.getElementById('canvas'); // output canvas to draw both webcam and detection results
+
+      async function drawLoop() { // main screen refresh loop
+        const interpolated = human.next(); // get smoothened result using last-known results which are continously updated based on input webcam video
+        human.draw.canvas(human.webcam.element, canvas); // draw webcam video to screen canvas // better than using procesed image as this loop happens faster than processing loop
+        await human.draw.all(canvas, interpolated); // draw labels, boxes, lines, etc.
+        setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 1000/30 ~ 30 fps
+      }
+
+      async function main() { // main entry point
+        document.getElementById('log').innerHTML = `human version: ${human.version} | tfjs version: ${human.tf.version['tfjs-core']}<br>platform: ${human.env.platform} | agent ${human.env.agent}`;
+        await human.webcam.start({ crop: true }); // find webcam and start it
+        human.video(human.webcam.element); // instruct human to continously detect video frames
+        canvas.width = human.webcam.width; // set canvas resolution to input webcam native resolution
+        canvas.height = human.webcam.height;
+        canvas.onclick = async () => { // pause when clicked on screen and resume on next click
+          if (human.webcam.paused) await human.webcam.play();
+          else human.webcam.pause();
+        };
+        await drawLoop(); // start draw loop
+      }
+
+      window.onload = main;
+    </script>  
+  </body>
+</html>

dist/tfjs.esm.d.ts

@@ -1,5 +1,8 @@
+/* eslint-disable import/no-unresolved */
 /* eslint-disable import/no-extraneous-dependencies */
 
+export * from 'types/tfjs.esm';
+
 export declare const version: {
   'tfjs-core': string;
   'tfjs-backend-cpu': string;
@@ -17,4 +20,6 @@ export * from '@tensorflow/tfjs-layers';
 export * from '@tensorflow/tfjs-backend-cpu';
 export * from '@tensorflow/tfjs-backend-wasm';
 export * from '@tensorflow/tfjs-backend-webgl';
-export {};
+export * from '@tensorflow/tfjs-backend-webgpu';
+export * from '@tensorflow/tfjs-node';
+export * from '@tensorflow/tfjs-node-gpu';

dist/tfjs.version.js

@@ -4,4 +4,4 @@
   author: <https://github.com/vladmandic>'
 */
 
-var e="3.20.0";var s="3.20.0";var t="3.20.0";var i="3.20.0";var n="3.20.0";var r="3.20.0";var l="3.20.0";var V={tfjs:e,"tfjs-core":s,"tfjs-data":t,"tfjs-layers":i,"tfjs-converter":n,"tfjs-backend-webgl":r,"tfjs-backend-wasm":l};export{V as version};
+var e="4.22.0";var s="4.22.0";var t="4.22.0";var n="4.22.0";var r="4.22.0";var i="4.22.0";var h={tfjs:e,"tfjs-core":e,"tfjs-converter":s,"tfjs-backend-cpu":t,"tfjs-backend-webgl":n,"tfjs-backend-wasm":r,"tfjs-backend-webgpu":i};export{h as version};

models/README.md

@@ -3,308 +3,3 @@
 For details see Wiki:
 
 - [**List of Models & Credits**](https://github.com/vladmandic/human/wiki/Models)
-
-## Model signatures:
-
-```js
-INFO:  graph model: /home/vlado/dev/human/models/iris.json
-INFO:  created on: 2020-10-12T18:46:47.060Z
-INFO:  metadata: { generatedBy: 'https://github.com/google/mediapipe', convertedBy: 'https://github.com/vladmandic', version: undefined }
-INFO:  model inputs based on signature
- { name: 'input_1:0', dtype: 'DT_FLOAT', shape: [ -1, 64, 64, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'Identity:0', dytpe: 'DT_FLOAT', shape: [ -1, 1, 1, 228 ] }
-INFO:  tensors: 191
-DATA:  weights: {
-  files: [ 'iris.bin' ],
-  size: { disk: 2599092, memory: 2599092 },
-  count: { total: 191, float32: 189, int32: 2 },
-  quantized: { none: 191 },
-  values: { total: 649773, float32: 649764, int32: 9 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder', 'Identity' ],
-  convolution: [ '_FusedConv2D', 'DepthwiseConv2dNative', 'MaxPool' ],
-  arithmetic: [ 'AddV2' ],
-  basic_math: [ 'Prelu' ],
-  transformation: [ 'Pad' ],
-  slice_join: [ 'ConcatV2' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/facemesh.json
-INFO:  created on: 2020-10-12T18:46:46.944Z
-INFO:  metadata: { generatedBy: 'https://github.com/google/mediapipe', convertedBy: 'https://github.com/vladmandic', version: undefined }
-INFO:  model inputs based on signature
- { name: 'input_1:0', dtype: 'DT_FLOAT', shape: [ 1, 192, 192, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'Identity_1:0', dytpe: 'DT_FLOAT', shape: [ 1, 266 ] }
- { id: 1, name: 'Identity_2:0', dytpe: 'DT_FLOAT', shape: [ 1, 1 ] }
- { id: 2, name: 'Identity:0', dytpe: 'DT_FLOAT', shape: [ 1, 1404 ] }
-INFO:  tensors: 118
-DATA:  weights: {
-  files: [ 'facemesh.bin' ],
-  size: { disk: 2955780, memory: 2955780 },
-  count: { total: 118, float32: 114, int32: 4 },
-  quantized: { none: 118 },
-  values: { total: 738945, float32: 738919, int32: 26 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Placeholder', 'Const', 'NoOp', 'Identity' ],
-  convolution: [ '_FusedConv2D', 'DepthwiseConv2dNative', 'MaxPool' ],
-  arithmetic: [ 'AddV2' ],
-  basic_math: [ 'Prelu', 'Sigmoid' ],
-  transformation: [ 'Pad', 'Reshape' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/emotion.json
-INFO:  created on: 2020-11-05T20:11:29.740Z
-INFO:  metadata: { generatedBy: 'https://github.com/oarriaga/face_classification', convertedBy: 'https://github.com/vladmandic', version: undefined }
-INFO:  model inputs based on signature
- { name: 'input_1:0', dtype: 'DT_FLOAT', shape: [ -1, 64, 64, 1 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'Identity:0', dytpe: 'DT_FLOAT', shape: [ -1, 7 ] }
-INFO:  tensors: 23
-DATA:  weights: {
-  files: [ 'emotion.bin' ],
-  size: { disk: 820516, memory: 820516 },
-  count: { total: 23, float32: 22, int32: 1 },
-  quantized: { none: 23 },
-  values: { total: 205129, float32: 205127, int32: 2 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder', 'Identity' ],
-  convolution: [ '_FusedConv2D', 'DepthwiseConv2dNative', 'MaxPool' ],
-  arithmetic: [ 'AddV2' ],
-  basic_math: [ 'Relu' ],
-  reduction: [ 'Mean' ],
-  normalization: [ 'Softmax' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/faceres.json
-INFO:  created on: 2021-03-21T14:12:59.863Z
-INFO:  metadata: { generatedBy: 'https://github.com/HSE-asavchenko/HSE_FaceRec_tf', convertedBy: 'https://github.com/vladmandic', version: undefined }
-INFO:  model inputs based on signature
- { name: 'input_1', dtype: 'DT_FLOAT', shape: [ -1, 224, 224, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'gender_pred/Sigmoid:0', dytpe: 'DT_FLOAT', shape: [ 1, 1 ] }
- { id: 1, name: 'global_pooling/Mean', dytpe: 'DT_FLOAT', shape: [ 1, 1024 ] }
- { id: 2, name: 'age_pred/Softmax:0', dytpe: 'DT_FLOAT', shape: [ 1, 100 ] }
-INFO:  tensors: 128
-DATA:  weights: {
-  files: [ 'faceres.bin' ],
-  size: { disk: 6978814, memory: 13957620 },
-  count: { total: 128, float32: 127, int32: 1 },
-  quantized: { float16: 127, none: 1 },
-  values: { total: 3489405, float32: 3489403, int32: 2 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder' ],
-  convolution: [ 'Conv2D', 'DepthwiseConv2dNative' ],
-  arithmetic: [ 'Add', 'Minimum', 'Maximum', 'Mul' ],
-  basic_math: [ 'Relu', 'Sigmoid' ],
-  reduction: [ 'Mean' ],
-  matrices: [ '_FusedMatMul' ],
-  normalization: [ 'Softmax' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/blazeface.json
-INFO:  created on: 2020-10-15T19:57:26.419Z
-INFO:  metadata: { generatedBy: 'https://github.com/google/mediapipe', convertedBy: 'https://github.com/vladmandic', version: undefined }
-INFO:  model inputs based on signature
- { name: 'input:0', dtype: 'DT_FLOAT', shape: [ 1, 256, 256, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'Identity_3:0', dytpe: 'DT_FLOAT', shape: [ 1, 384, 16 ] }
- { id: 1, name: 'Identity:0', dytpe: 'DT_FLOAT', shape: [ 1, 512, 1 ] }
- { id: 2, name: 'Identity_1:0', dytpe: 'DT_FLOAT', shape: [ 1, 384, 1 ] }
- { id: 3, name: 'Identity_2:0', dytpe: 'DT_FLOAT', shape: [ 1, 512, 16 ] }
-INFO:  tensors: 112
-DATA:  weights: {
-  files: [ 'blazeface.bin' ],
-  size: { disk: 538928, memory: 538928 },
-  count: { total: 112, float32: 106, int32: 6 },
-  quantized: { none: 112 },
-  values: { total: 134732, float32: 134704, int32: 28 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder', 'Identity' ],
-  convolution: [ '_FusedConv2D', 'DepthwiseConv2dNative', 'MaxPool' ],
-  arithmetic: [ 'AddV2' ],
-  basic_math: [ 'Relu' ],
-  transformation: [ 'Pad', 'Reshape' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/mb3-centernet.json
-INFO:  created on: 2021-05-19T11:50:13.013Z
-INFO:  metadata: { generatedBy: 'https://github.com/610265158/mobilenetv3_centernet', convertedBy: 'https://github.com/vladmandic', version: undefined }
-INFO:  model inputs based on signature
- { name: 'tower_0/images', dtype: 'DT_FLOAT', shape: [ 1, 512, 512, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'tower_0/wh', dytpe: 'DT_FLOAT', shape: [ 1, 128, 128, 4 ] }
- { id: 1, name: 'tower_0/keypoints', dytpe: 'DT_FLOAT', shape: [ 1, 128, 128, 80 ] }
- { id: 2, name: 'tower_0/detections', dytpe: 'DT_FLOAT', shape: [ 1, 100, 6 ] }
-INFO:  tensors: 267
-DATA:  weights: {
-  files: [ 'mb3-centernet.bin' ],
-  size: { disk: 4030290, memory: 8060260 },
-  count: { total: 267, float32: 227, int32: 40 },
-  quantized: { float16: 227, none: 40 },
-  values: { total: 2015065, float32: 2014985, int32: 80 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder', 'Identity' ],
-  convolution: [ '_FusedConv2D', 'FusedDepthwiseConv2dNative', 'DepthwiseConv2dNative', 'Conv2D', 'MaxPool' ],
-  arithmetic: [ 'Mul', 'Add', 'FloorDiv', 'FloorMod', 'Sub' ],
-  basic_math: [ 'Relu6', 'Relu', 'Sigmoid' ],
-  reduction: [ 'Mean' ],
-  image: [ 'ResizeBilinear' ],
-  slice_join: [ 'ConcatV2', 'GatherV2', 'StridedSlice' ],
-  transformation: [ 'Reshape', 'Cast', 'ExpandDims' ],
-  logical: [ 'Equal' ],
-  evaluation: [ 'TopKV2' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/movenet-lightning.json
-INFO:  created on: 2021-05-29T12:26:32.994Z
-INFO:  metadata: { generatedBy: 'https://tfhub.dev/google/movenet/singlepose/lightning/4', convertedBy: 'https://github.com/vladmandic', version: undefined }
-INFO:  model inputs based on signature
- { name: 'input:0', dtype: 'DT_INT32', shape: [ 1, 192, 192, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'Identity:0', dytpe: 'DT_FLOAT', shape: [ 1, 1, 17, 3 ] }
-INFO:  tensors: 180
-DATA:  weights: {
-  files: [ 'movenet-lightning.bin' ],
-  size: { disk: 4650216, memory: 9300008 },
-  count: { total: 180, int32: 31, float32: 149 },
-  quantized: { none: 31, float16: 149 },
-  values: { total: 2325002, int32: 106, float32: 2324896 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder', 'Identity' ],
-  transformation: [ 'Cast', 'ExpandDims', 'Squeeze', 'Reshape' ],
-  slice_join: [ 'Unpack', 'Pack', 'GatherNd', 'ConcatV2' ],
-  arithmetic: [ 'Sub', 'Mul', 'AddV2', 'FloorDiv', 'SquaredDifference', 'RealDiv' ],
-  convolution: [ '_FusedConv2D', 'FusedDepthwiseConv2dNative', 'DepthwiseConv2dNative' ],
-  image: [ 'ResizeBilinear' ],
-  basic_math: [ 'Sigmoid', 'Sqrt' ],
-  reduction: [ 'ArgMax' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/selfie.json
-INFO:  created on: 2021-06-04T13:46:56.904Z
-INFO:  metadata: { generatedBy: 'https://github.com/PINTO0309/PINTO_model_zoo/tree/main/109_Selfie_Segmentation', convertedBy: 'https://github.com/vladmandic', version: '561.undefined' }
-INFO:  model inputs based on signature
- { name: 'input_1:0', dtype: 'DT_FLOAT', shape: [ 1, 256, 256, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'activation_10:0', dytpe: 'DT_FLOAT', shape: [ 1, 256, 256, 1 ] }
-INFO:  tensors: 136
-DATA:  weights: {
-  files: [ 'selfie.bin' ],
-  size: { disk: 212886, memory: 425732 },
-  count: { total: 136, int32: 4, float32: 132 },
-  quantized: { none: 4, float16: 132 },
-  values: { total: 106433, int32: 10, float32: 106423 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder' ],
-  convolution: [ 'Conv2D', 'DepthwiseConv2dNative', 'AvgPool', 'Conv2DBackpropInput' ],
-  arithmetic: [ 'Add', 'Mul', 'AddV2', 'AddN' ],
-  basic_math: [ 'Relu6', 'Relu', 'Sigmoid' ],
-  image: [ 'ResizeBilinear' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/handtrack.json
-INFO:  created on: 2021-09-21T12:09:47.583Z
-INFO:  metadata: { generatedBy: 'https://github.com/victordibia/handtracking', convertedBy: 'https://github.com/vladmandic', version: '561.undefined' }
-INFO:  model inputs based on signature
- { name: 'input_tensor:0', dtype: 'DT_UINT8', shape: [ 1, 320, 320, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'Identity_2:0', dytpe: 'DT_FLOAT', shape: [ 1, 100 ] }
- { id: 1, name: 'Identity_4:0', dytpe: 'DT_FLOAT', shape: [ 1, 100 ] }
- { id: 2, name: 'Identity_6:0', dytpe: 'DT_FLOAT', shape: [ 1, 12804, 4 ] }
- { id: 3, name: 'Identity_1:0', dytpe: 'DT_FLOAT', shape: [ 1, 100, 4 ] }
- { id: 4, name: 'Identity_3:0', dytpe: 'DT_FLOAT', shape: [ 1, 100, 8 ] }
- { id: 5, name: 'Identity_5:0', dytpe: 'DT_FLOAT', shape: [ 1 ] }
- { id: 6, name: 'Identity:0', dytpe: 'DT_FLOAT', shape: [ 1, 100 ] }
- { id: 7, name: 'Identity_7:0', dytpe: 'DT_FLOAT', shape: [ 1, 12804, 8 ] }
-INFO:  tensors: 619
-DATA:  weights: {
-  files: [ 'handtrack.bin' ],
-  size: { disk: 2964837, memory: 11846016 },
-  count: { total: 619, int32: 347, float32: 272 },
-  quantized: { none: 347, uint8: 272 },
-  values: { total: 2961504, int32: 1111, float32: 2960393 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder', 'Identity', 'Shape', 'NoOp' ],
-  control: [ 'TensorListReserve', 'Enter', 'TensorListFromTensor', 'Merge', 'LoopCond', 'Switch', 'Exit', 'TensorListStack', 'NextIteration', 'TensorListSetItem', 'TensorListGetItem' ],
-  logical: [ 'Less', 'LogicalAnd', 'Select', 'Greater', 'GreaterEqual' ],
-  convolution: [ '_FusedConv2D', 'FusedDepthwiseConv2dNative', 'DepthwiseConv2dNative' ],
-  arithmetic: [ 'AddV2', 'Mul', 'Sub', 'Minimum', 'Maximum' ],
-  transformation: [ 'Cast', 'ExpandDims', 'Squeeze', 'Reshape', 'Pad' ],
-  slice_join: [ 'Unpack', 'StridedSlice', 'Pack', 'ConcatV2', 'Slice', 'GatherV2', 'Split' ],
-  image: [ 'ResizeBilinear' ],
-  basic_math: [ 'Reciprocal', 'Sigmoid', 'Exp' ],
-  matrices: [ 'Transpose' ],
-  dynamic: [ 'NonMaxSuppressionV5', 'Where' ],
-  creation: [ 'Fill', 'Range' ],
-  evaluation: [ 'TopKV2' ],
-  reduction: [ 'Sum' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/antispoof.json
-INFO:  created on: 2021-10-13T14:20:27.100Z
-INFO:  metadata: { generatedBy: 'https://www.kaggle.com/anku420/fake-face-detection', convertedBy: 'https://github.com/vladmandic', version: '716.undefined' }
-INFO:  model inputs based on signature
- { name: 'conv2d_input', dtype: 'DT_FLOAT', shape: [ -1, 128, 128, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'activation_4', dytpe: 'DT_FLOAT', shape: [ -1, 1 ] }
-INFO:  tensors: 11
-DATA:  weights: {
-  files: [ 'antispoof.bin' ],
-  size: { disk: 853098, memory: 1706188 },
-  count: { total: 11, float32: 10, int32: 1 },
-  quantized: { float16: 10, none: 1 },
-  values: { total: 426547, float32: 426545, int32: 2 }
-}
-DATA:  kernel ops: { graph: [ 'Const', 'Placeholder', 'Identity' ], convolution: [ '_FusedConv2D', 'MaxPool' ], basic_math: [ 'Relu', 'Sigmoid' ], transformation: [ 'Reshape' ], matrices: [ '_FusedMatMul' ] }
-INFO:  graph model: /home/vlado/dev/human/models/handlandmark-full.json
-INFO:  created on: 2021-10-31T12:27:49.343Z
-INFO:  metadata: { generatedBy: 'https://github.com/google/mediapipe', convertedBy: 'https://github.com/vladmandic', version: '808.undefined' }
-INFO:  model inputs based on signature
- { name: 'input_1', dtype: 'DT_FLOAT', shape: [ 1, 224, 224, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'Identity_3:0', dytpe: 'DT_FLOAT', shape: [ 1, 63 ] }
- { id: 1, name: 'Identity:0', dytpe: 'DT_FLOAT', shape: [ 1, 63 ] }
- { id: 2, name: 'Identity_1:0', dytpe: 'DT_FLOAT', shape: [ 1, 1 ] }
- { id: 3, name: 'Identity_2:0', dytpe: 'DT_FLOAT', shape: [ 1, 1 ] }
-INFO:  tensors: 103
-DATA:  weights: {
-  files: [ 'handlandmark-full.bin' ],
-  size: { disk: 5431368, memory: 10862728 },
-  count: { total: 103, float32: 102, int32: 1 },
-  quantized: { float16: 102, none: 1 },
-  values: { total: 2715682, float32: 2715680, int32: 2 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder', 'Identity' ],
-  convolution: [ 'Conv2D', 'DepthwiseConv2dNative' ],
-  arithmetic: [ 'AddV2', 'AddN' ],
-  basic_math: [ 'Relu6', 'Sigmoid' ],
-  reduction: [ 'Mean' ],
-  matrices: [ '_FusedMatMul' ]
-}
-INFO:  graph model: /home/vlado/dev/human/models/liveness.json
-INFO:  created on: 2021-11-09T12:39:11.760Z
-INFO:  metadata: { generatedBy: 'https://github.com/leokwu/livenessnet', convertedBy: 'https://github.com/vladmandic', version: '808.undefined' }
-INFO:  model inputs based on signature
- { name: 'conv2d_1_input', dtype: 'DT_FLOAT', shape: [ -1, 32, 32, 3 ] }
-INFO:  model outputs based on signature
- { id: 0, name: 'activation_6', dytpe: 'DT_FLOAT', shape: [ -1, 2 ] }
-INFO:  tensors: 23
-DATA:  weights: {
-  files: [ 'liveness.bin' ],
-  size: { disk: 592976, memory: 592976 },
-  count: { total: 23, float32: 22, int32: 1 },
-  quantized: { none: 23 },
-  values: { total: 148244, float32: 148242, int32: 2 }
-}
-DATA:  kernel ops: {
-  graph: [ 'Const', 'Placeholder', 'Identity' ],
-  convolution: [ '_FusedConv2D', 'MaxPool' ],
-  arithmetic: [ 'Mul', 'Add', 'AddV2' ],
-  transformation: [ 'Reshape' ],
-  matrices: [ '_FusedMatMul' ],
-  normalization: [ 'Softmax' ]
-}
-```

models/models.json

@@ -1,38 +1,38 @@
 {
   "antispoof": 853098,
   "blazeface": 538928,
+  "centernet": 4030290,
   "emotion": 820516,
   "facemesh": 1477958,
   "faceres": 6978814,
-  "handlandmark-full": 5431368,
+  "handlandmark-lite": 2023432,
   "handtrack": 2964837,
   "iris": 2599092,
   "liveness": 592976,
-  "mb3-centernet": 4030290,
   "models": 0,
   "movenet-lightning": 4650216,
-  "selfie": 212886,
+  "affectnet-mobilenet": 6920630,
   "age": 161240,
   "blazeface-back": 538928,
   "blazeface-front": 402048,
-  "blazepose-detector2d": 7499400,
-  "blazepose-detector3d": 5928856,
-  "blazepose-full": 6338290,
-  "blazepose-heavy": 27501554,
-  "blazepose-lite": 2725490,
+  "blazepose-detector": 5928856,
+  "blazepose-full": 6339202,
+  "blazepose-heavy": 27502466,
+  "blazepose-lite": 2726402,
   "efficientpose": 5651240,
   "faceboxes": 2013002,
-  "facemesh-attention-alt": 2387598,
+  "facemesh-attention-pinto": 2387598,
   "facemesh-attention": 2382414,
   "facemesh-detection-full": 1026192,
   "facemesh-detection-short": 201268,
-  "facemesh-orig": 2955780,
   "faceres-deep": 13957620,
+  "gear-e1": 112438,
+  "gear-e2": 112438,
   "gear": 1498916,
   "gender-ssrnet-imdb": 161236,
   "gender": 201808,
   "handdetect": 3515612,
-  "handlandmark-lite": 2023432,
+  "handlandmark-full": 5431368,
   "handlandmark-sparse": 5286322,
   "handskeleton": 5502280,
   "meet": 372228,
@@ -42,7 +42,8 @@
   "movenet-thunder": 12477112,
   "nanodet": 7574558,
   "posenet": 5032780,
-  "blazepose-detect": 5928804,
+  "rvm": 3739355,
+  "selfie": 212886,
   "anti-spoofing": 853098,
   "efficientpose-i-lite": 2269064,
   "efficientpose-ii-lite": 5651240,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vladmandic/human",
-  "version": "2.10.1",
+  "version": "3.3.6",
   "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
   "sideEffects": false,
   "main": "dist/human.node.js",
@@ -8,11 +8,24 @@
   "browser": "dist/human.esm.js",
   "types": "types/human.d.ts",
   "exports": {
-    "require": "./dist/human.node.js",
-    "import": "./dist/human.esm.js",
+    "node": "./dist/human.node.js",
     "script": "./dist/human.js",
     "module": "./dist/human.esm.js",
-    "types": "./types/human.d.ts"
+    "types": "./types/human.d.ts",
+    "dist/human": "./dist/human.js",
+    "dist/human.js": "./dist/human.js",
+    "dist/human.esm": "./dist/human.esm.js",
+    "dist/human.esm.js": "./dist/human.esm.js",
+    "dist/human.esm-nobundle": "./dist/human.esm-nobundle.js",
+    "dist/human.esm-nobundle.js": "./dist/human.esm-nobundle.js",
+    "dist/human.node": "./dist/human.node.js",
+    "dist/human.node.js": "./dist/human.node.js",
+    "dist/human.node-wasm": "./dist/human.node-wasm.js",
+    "dist/human.node-wasm.js": "./dist/human.node-wasm.js",
+    "dist/human.node-gpu": "./dist/human.node-gpu.js",
+    "dist/human.node-gpu.js": "./dist/human.node-gpu.js",
+    "require": "./dist/human.node.js",
+    "import": "./dist/human.esm.js"
   },
   "author": "Vladimir Mandic <mandic00@live.com>",
   "bugs": {
@@ -33,7 +46,7 @@
     "clean": "build --profile clean",
     "build": "rimraf test/build.log && node build.js",
     "test": "node --no-warnings --unhandled-rejections=strict --trace-uncaught test/node.js",
-    "lint": "eslint *.json *.js src demo test models",
+    "lint": "eslint *.json *.js src demo test models wiki",
     "scan": "npx auditjs@latest ossi --dev --quiet"
   },
   "keywords": [
@@ -61,43 +74,40 @@
     "tensorflow"
   ],
   "devDependencies": {
-    "@html-eslint/eslint-plugin": "^0.13.2",
-    "@html-eslint/parser": "^0.13.2",
-    "@microsoft/api-extractor": "^7.30.0",
-    "@tensorflow/tfjs": "^3.20.0",
-    "@tensorflow/tfjs-backend-cpu": "^3.20.0",
-    "@tensorflow/tfjs-backend-wasm": "^3.20.0",
-    "@tensorflow/tfjs-backend-webgl": "^3.20.0",
-    "@tensorflow/tfjs-backend-webgpu": "0.0.1-alpha.13",
-    "@tensorflow/tfjs-converter": "^3.20.0",
-    "@tensorflow/tfjs-core": "^3.20.0",
-    "@tensorflow/tfjs-data": "^3.20.0",
-    "@tensorflow/tfjs-layers": "^3.20.0",
-    "@tensorflow/tfjs-node": "^3.20.0",
-    "@tensorflow/tfjs-node-gpu": "^3.20.0",
-    "@tensorflow/tfjs-tflite": "0.0.1-alpha.8",
-    "@types/node": "^18.7.15",
-    "@types/offscreencanvas": "^2019.7.0",
-    "@typescript-eslint/eslint-plugin": "^5.36.2",
-    "@typescript-eslint/parser": "^5.36.2",
-    "@vladmandic/build": "^0.7.11",
-    "@vladmandic/pilogger": "^0.4.6",
+    "@html-eslint/eslint-plugin": "^0.46.1",
+    "@html-eslint/parser": "^0.46.0",
+    "@microsoft/api-extractor": "^7.52.11",
+    "@tensorflow/tfjs-backend-cpu": "^4.22.0",
+    "@tensorflow/tfjs-backend-wasm": "^4.22.0",
+    "@tensorflow/tfjs-backend-webgl": "^4.22.0",
+    "@tensorflow/tfjs-backend-webgpu": "4.22.0",
+    "@tensorflow/tfjs-converter": "^4.22.0",
+    "@tensorflow/tfjs-core": "^4.22.0",
+    "@tensorflow/tfjs-data": "^4.22.0",
+    "@tensorflow/tfjs-layers": "^4.22.0",
+    "@tensorflow/tfjs-node": "^4.22.0",
+    "@tensorflow/tfjs-node-gpu": "^4.22.0",
+    "@types/emscripten": "^1.40.1",
+    "@types/node": "^24.3.0",
+    "@types/offscreencanvas": "^2019.7.3",
+    "@typescript-eslint/eslint-plugin": "^8.41.0",
+    "@typescript-eslint/parser": "^8.41.0",
+    "@vladmandic/build": "^0.10.3",
+    "@vladmandic/pilogger": "^0.5.2",
     "@vladmandic/tfjs": "github:vladmandic/tfjs",
-    "@webgpu/types": "^0.1.21",
-    "canvas": "^2.10.0",
-    "esbuild": "^0.15.7",
-    "eslint": "8.23.0",
+    "canvas": "^3.2.0",
+    "esbuild": "^0.25.9",
+    "eslint": "8.57.0",
     "eslint-config-airbnb-base": "^15.0.0",
-    "eslint-plugin-html": "^7.1.0",
-    "eslint-plugin-import": "^2.26.0",
-    "eslint-plugin-json": "^3.1.0",
+    "eslint-plugin-html": "^8.1.3",
+    "eslint-plugin-import": "^2.32.0",
+    "eslint-plugin-json": "^4.0.1",
+    "eslint-plugin-markdown": "^5.1.0",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^6.0.1",
-    "node-fetch": "^3.2.10",
-    "rimraf": "^3.0.2",
-    "seedrandom": "^3.0.5",
-    "tslib": "^2.4.0",
-    "typedoc": "0.23.14",
-    "typescript": "4.8.2"
+    "eslint-plugin-promise": "^7.2.1",
+    "rimraf": "^6.0.1",
+    "tslib": "^2.8.1",
+    "typedoc": "0.28.11",
+    "typescript": "5.9.2"
   }
 }

samples/index.html

@@ -35,7 +35,7 @@
     <div class="text">Human Examples Gallery</div>
     <div id="strip" class="strip"></div>
     <div class="image-container">
-      <img id="image" src="data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=" alt="" class="image-zoomwidth" />
+      <img id="image" src="data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=" alt="" class="image-zoomwidth"></img>
       <div class="arrow" id="arrow-left" style="left: 10px">&lt;</div>
       <div class="arrow" id="arrow-right" style="right: 10px">&gt;</div>
     </div>