update todo and docs

CHANGELOG.md

@@ -9,11 +9,10 @@
   
 ## Changelog
   
-### **HEAD -> main** 2021/09/21 mandic00@live.com
-
-
-### **origin/main** 2021/09/20 mandic00@live.com
+### **HEAD -> main** 2021/09/22 mandic00@live.com
 
+- prototype handtracking
+- automated browser tests
 - support for dynamic backend switching
 - initial automated browser tests
 - enhanced automated test coverage

TODO.md

@@ -2,14 +2,31 @@
 
 ## Work in Progress
 
-WebGL shader optimizations for faster load and initial detection
+<br>
 
-- Fix shader packing: <https://github.com/tensorflow/tfjs/issues/5343>
-- Add and benchmark WGSL for WebGPU
+### Handtrack
+
+- Finish implementation
+- Set defaults and image sizes
+- Optimize model
+- Add tests
 
 <br>
 
-## Exploring
+### Segmentation
+
+- Implement `NodeJS` support
+- Test for leaks
+
+### Backends
+
+- Optimize shader packing for WebGL backend:  
+  <https://github.com/tensorflow/tfjs/issues/5343>  
+- Add and benchmark WGSL for WebGPU  
+
+<br>
+
+### Exploring
 
 - Optical Flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
 - TFLite Models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>

package.json

@@ -66,14 +66,14 @@
     "@tensorflow/tfjs-layers": "^3.9.0",
     "@tensorflow/tfjs-node": "^3.9.0",
     "@tensorflow/tfjs-node-gpu": "^3.9.0",
-    "@types/node": "^16.9.4",
-    "@typescript-eslint/eslint-plugin": "^4.31.1",
-    "@typescript-eslint/parser": "^4.31.1",
-    "@vladmandic/build": "^0.5.2",
+    "@types/node": "^16.9.6",
+    "@typescript-eslint/eslint-plugin": "^4.31.2",
+    "@typescript-eslint/parser": "^4.31.2",
+    "@vladmandic/build": "^0.5.3",
     "@vladmandic/pilogger": "^0.3.3",
     "canvas": "^2.8.0",
     "dayjs": "^1.10.7",
-    "esbuild": "^0.12.28",
+    "esbuild": "^0.13.0",
     "eslint": "^7.32.0",
     "eslint-config-airbnb-base": "^14.2.1",
     "eslint-plugin-import": "^2.24.2",

src/config.ts

@@ -1,6 +1,7 @@
 /* eslint-disable indent */
 /* eslint-disable no-multi-spaces */
 
+/** Dectector part of face configuration */
 export interface FaceDetectorConfig {
   modelPath: string,
   rotation: boolean,
@@ -11,16 +12,21 @@ export interface FaceDetectorConfig {
   return: boolean,
 }
 
+/** Mesh part of face configuration */
 export interface FaceMeshConfig {
   enabled: boolean,
   modelPath: string,
 }
 
+/** Iris part of face configuration */
 export interface FaceIrisConfig {
   enabled: boolean,
   modelPath: string,
 }
 
+/** Description or face embedding part of face configuration
+ * - also used by age and gender detection
+ */
 export interface FaceDescriptionConfig {
   enabled: boolean,
   modelPath: string,
@@ -28,6 +34,7 @@ export interface FaceDescriptionConfig {
   minConfidence: number,
 }
 
+/** Emotion part of face configuration */
 export interface FaceEmotionConfig {
   enabled: boolean,
   minConfidence: number,
@@ -37,6 +44,7 @@ export interface FaceEmotionConfig {
 
 /** Controlls and configures all face-specific options:
  * - face detection, face mesh detection, age, gender, emotion detection and face description
+ *
  * Parameters:
  * - enabled: true/false
  * - modelPath: path for each of face models
@@ -56,10 +64,15 @@ export interface FaceConfig {
 }
 
 /** Controlls and configures all body detection specific options
+ *
+ * Parameters:
  * - enabled: true/false
  * - modelPath: body pose model, can be absolute path or relative to modelBasePath
  * - minConfidence: threshold for discarding a prediction
  * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance
+ *
+ * Changing `modelPath` will change module responsible for hand detection and tracking
+ * Allowed values are 'posenet.json', 'blazepose.json', 'efficientpose.json', 'movenet-lightning.json', 'movenet-thunder.json', 'movenet-multipose.json'
 */
 export interface BodyConfig {
   enabled: boolean,
@@ -70,6 +83,8 @@ export interface BodyConfig {
 }
 
 /** Controlls and configures all hand detection specific options
+ *
+ * Parameters:
  * - enabled: true/false
  * - landmarks: detect hand landmarks or just hand boundary box
  * - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath
@@ -77,6 +92,9 @@ export interface BodyConfig {
  * - iouThreshold: ammount of overlap between two detected objects before one object is removed
  * - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance
  * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
+ *
+ * Changing `detector.modelPath` will change module responsible for hand detection and tracking
+ * Allowed values are `handdetect.json` and `handtrack.json`
 */
 export interface HandConfig {
   enabled: boolean,
@@ -100,6 +118,9 @@ export interface HandConfig {
  * - minConfidence: minimum score that detection must have to return as valid object
  * - iouThreshold: ammount of overlap between two detected objects before one object is removed
  * - maxDetected: maximum number of detections to return
+ *
+ * Changing `modelPath` will change module responsible for hand detection and tracking
+ * Allowed values are `mb3-centernet.json` and `nanodet.json`
 */
 export interface ObjectConfig {
   enabled: boolean,
@@ -119,6 +140,10 @@ export interface ObjectConfig {
  * - enabled: true/false
  * - modelPath: object detection model, can be absolute path or relative to modelBasePath
  * - blur: blur segmentation output by <number> pixels for more realistic image
+ *
+ * Changing `modelPath` will change module responsible for hand detection and tracking
+ * Allowed values are `selfie.json` and `meet.json`
+
 */
 export interface SegmentationConfig {
   enabled: boolean,
@@ -127,7 +152,8 @@ export interface SegmentationConfig {
 }
 
 /** Run input through image filters before inference
- * - image filters run with near-zero latency as they are executed on the GPU
+ * - available only in Browser environments
+ * - image filters run with near-zero latency as they are executed on the GPU using WebGL
 */
 export interface FilterConfig {
   enabled: boolean,
@@ -202,7 +228,6 @@ export interface Config {
 
   /** What to use for `human.warmup()`
    * - warmup pre-initializes all models for faster inference but can take significant time on startup
-   * - only used for `webgl` and `humangl` backends
   */
   warmup: 'none' | 'face' | 'full' | 'body',
   // warmup: string;
@@ -218,9 +243,6 @@ export interface Config {
   */
   cacheSensitivity: number;
 
-  /** Yield to main thread periodically */
-  yield: boolean;
-
   /** Internal Variable */
   skipFrame: boolean;
 
@@ -249,7 +271,8 @@ export interface Config {
  */
 const config: Config = {
   backend: '',               // select tfjs backend to use, leave empty to use default backend
-                             // can be 'webgl', 'wasm', 'cpu', or 'humangl' which is a custom version of webgl
+                             // for browser environments: 'webgl', 'wasm', 'cpu', or 'humangl' (which is a custom version of webgl)
+                             // for nodejs environments: 'tensorflow', 'wasm', 'cpu'
                              // default set to `humangl` for browsers and `tensorflow` for nodejs
   modelBasePath: '',         // base path for all models
                              // default set to `../models/` for browsers and `file://models/` for nodejs
@@ -264,7 +287,6 @@ const config: Config = {
   cacheSensitivity: 0.75,    // cache sensitivity
                              // values 0..1 where 0.01 means reset cache if input changed more than 1%
                              // set to 0 to disable caching
-  yield: false,              // yield to main thread periodically
   skipFrame: false,          // internal & dynamic
   filter: {                  // run input through image filters before inference
                              // image filters run with near-zero latency as they are executed on the GPU
@@ -376,6 +398,7 @@ const config: Config = {
     landmarks: true,         // detect hand landmarks or just hand boundary box
     detector: {
       modelPath: 'handdetect.json',  // hand detector model, can be absolute path or relative to modelBasePath
+                             // can be 'handdetect' or 'handtrack'
     },
     skeleton: {
       modelPath: 'handskeleton.json',  // hand skeleton model, can be absolute path or relative to modelBasePath

test/test-main.js

@@ -170,7 +170,7 @@ async function test(Human, inputConfig) {
   await human.load();
   const models = Object.keys(human.models).map((model) => ({ name: model, loaded: (human.models[model] !== null) }));
   const loaded = models.filter((model) => model.loaded);
-  if (models.length === 14 && loaded.length === 7) log('state', 'passed: models loaded', models.length, loaded.length);
+  if (models.length === 15 && loaded.length === 7) log('state', 'passed: models loaded', models.length, loaded.length);
   else log('error', 'failed: models loaded', models.length, loaded.length);
 
   // test warmup sequences
@@ -338,8 +338,8 @@ async function test(Human, inputConfig) {
 
   // test segmentation
   res = await human.segmentation(inputCanvas, inputCanvas);
-  if (!res || !res.width || !res.height) log('error', 'failed: segmentation', res);
-  else log('state', 'passed: segmentation', [res.width, res.height]);
+  if (!res || !res.data) log('error', 'failed: segmentation', res);
+  else log('state', 'passed: segmentation', [res.data.length]);
   human.env.Canvas = undefined;
 
   // tests end

wiki

@@ -1 +1 @@
-Subproject commit d293f4a20b640e6bc8485dc0f8a2c2147ce33073
+Subproject commit a0497b6d14059099b2764b8f70390f4b6af8db9f