diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d2f0fdd..c6d8de2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,10 @@ ## Changelog -### **HEAD -> main** 2021/09/21 mandic00@live.com - - -### **origin/main** 2021/09/20 mandic00@live.com +### **HEAD -> main** 2021/09/22 mandic00@live.com +- prototype handtracking +- automated browser tests - support for dynamic backend switching - initial automated browser tests - enhanced automated test coverage diff --git a/TODO.md b/TODO.md index b03b656e..057ff3ae 100644 --- a/TODO.md +++ b/TODO.md @@ -2,14 +2,31 @@ ## Work in Progress -WebGL shader optimizations for faster load and initial detection +
-- Fix shader packing: -- Add and benchmark WGSL for WebGPU +### Handtrack + +- Finish implementation +- Set defaults and image sizes +- Optimize model +- Add tests
-## Exploring +### Segmentation + +- Implement `NodeJS` support +- Test for leaks + +### Backends + +- Optimize shader packing for WebGL backend: + +- Add and benchmark WGSL for WebGPU + +
+ +### Exploring - Optical Flow: - TFLite Models: diff --git a/package.json b/package.json index 655c5753..6a32637b 100644 --- a/package.json +++ b/package.json @@ -66,14 +66,14 @@ "@tensorflow/tfjs-layers": "^3.9.0", "@tensorflow/tfjs-node": "^3.9.0", "@tensorflow/tfjs-node-gpu": "^3.9.0", - "@types/node": "^16.9.4", - "@typescript-eslint/eslint-plugin": "^4.31.1", - "@typescript-eslint/parser": "^4.31.1", - "@vladmandic/build": "^0.5.2", + "@types/node": "^16.9.6", + "@typescript-eslint/eslint-plugin": "^4.31.2", + "@typescript-eslint/parser": "^4.31.2", + "@vladmandic/build": "^0.5.3", "@vladmandic/pilogger": "^0.3.3", "canvas": "^2.8.0", "dayjs": "^1.10.7", - "esbuild": "^0.12.28", + "esbuild": "^0.13.0", "eslint": "^7.32.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-plugin-import": "^2.24.2", diff --git a/src/config.ts b/src/config.ts index b0c7694f..c8de876a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,7 @@ /* eslint-disable indent */ /* eslint-disable no-multi-spaces */ +/** Dectector part of face configuration */ export interface FaceDetectorConfig { modelPath: string, rotation: boolean, @@ -11,16 +12,21 @@ export interface FaceDetectorConfig { return: boolean, } +/** Mesh part of face configuration */ export interface FaceMeshConfig { enabled: boolean, modelPath: string, } +/** Iris part of face configuration */ export interface FaceIrisConfig { enabled: boolean, modelPath: string, } +/** Description or face embedding part of face configuration + * - also used by age and gender detection + */ export interface FaceDescriptionConfig { enabled: boolean, modelPath: string, @@ -28,6 +34,7 @@ export interface FaceDescriptionConfig { minConfidence: number, } +/** Emotion part of face configuration */ export interface FaceEmotionConfig { enabled: boolean, minConfidence: number, @@ -37,6 +44,7 @@ export interface FaceEmotionConfig { /** Controlls and configures all face-specific options: * - face detection, face mesh detection, age, gender, emotion detection and face description + * * Parameters: * - enabled: true/false * - modelPath: path for each of face models @@ -56,10 +64,15 @@ export interface FaceConfig { } /** Controlls and configures all body detection specific options + * + * Parameters: * - enabled: true/false * - modelPath: body pose model, can be absolute path or relative to modelBasePath * - minConfidence: threshold for discarding a prediction * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance + * + * Changing `modelPath` will change module responsible for hand detection and tracking + * Allowed values are 'posenet.json', 'blazepose.json', 'efficientpose.json', 'movenet-lightning.json', 'movenet-thunder.json', 'movenet-multipose.json' */ export interface BodyConfig { enabled: boolean, @@ -70,6 +83,8 @@ export interface BodyConfig { } /** Controlls and configures all hand detection specific options + * + * Parameters: * - enabled: true/false * - landmarks: detect hand landmarks or just hand boundary box * - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath @@ -77,6 +92,9 @@ export interface BodyConfig { * - iouThreshold: ammount of overlap between two detected objects before one object is removed * - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted + * + * Changing `detector.modelPath` will change module responsible for hand detection and tracking + * Allowed values are `handdetect.json` and `handtrack.json` */ export interface HandConfig { enabled: boolean, @@ -100,6 +118,9 @@ export interface HandConfig { * - minConfidence: minimum score that detection must have to return as valid object * - iouThreshold: ammount of overlap between two detected objects before one object is removed * - maxDetected: maximum number of detections to return + * + * Changing `modelPath` will change module responsible for hand detection and tracking + * Allowed values are `mb3-centernet.json` and `nanodet.json` */ export interface ObjectConfig { enabled: boolean, @@ -119,6 +140,10 @@ export interface ObjectConfig { * - enabled: true/false * - modelPath: object detection model, can be absolute path or relative to modelBasePath * - blur: blur segmentation output by pixels for more realistic image + * + * Changing `modelPath` will change module responsible for hand detection and tracking + * Allowed values are `selfie.json` and `meet.json` + */ export interface SegmentationConfig { enabled: boolean, @@ -127,7 +152,8 @@ export interface SegmentationConfig { } /** Run input through image filters before inference - * - image filters run with near-zero latency as they are executed on the GPU + * - available only in Browser environments + * - image filters run with near-zero latency as they are executed on the GPU using WebGL */ export interface FilterConfig { enabled: boolean, @@ -202,7 +228,6 @@ export interface Config { /** What to use for `human.warmup()` * - warmup pre-initializes all models for faster inference but can take significant time on startup - * - only used for `webgl` and `humangl` backends */ warmup: 'none' | 'face' | 'full' | 'body', // warmup: string; @@ -218,9 +243,6 @@ export interface Config { */ cacheSensitivity: number; - /** Yield to main thread periodically */ - yield: boolean; - /** Internal Variable */ skipFrame: boolean; @@ -249,7 +271,8 @@ export interface Config { */ const config: Config = { backend: '', // select tfjs backend to use, leave empty to use default backend - // can be 'webgl', 'wasm', 'cpu', or 'humangl' which is a custom version of webgl + // for browser environments: 'webgl', 'wasm', 'cpu', or 'humangl' (which is a custom version of webgl) + // for nodejs environments: 'tensorflow', 'wasm', 'cpu' // default set to `humangl` for browsers and `tensorflow` for nodejs modelBasePath: '', // base path for all models // default set to `../models/` for browsers and `file://models/` for nodejs @@ -264,7 +287,6 @@ const config: Config = { cacheSensitivity: 0.75, // cache sensitivity // values 0..1 where 0.01 means reset cache if input changed more than 1% // set to 0 to disable caching - yield: false, // yield to main thread periodically skipFrame: false, // internal & dynamic filter: { // run input through image filters before inference // image filters run with near-zero latency as they are executed on the GPU @@ -376,6 +398,7 @@ const config: Config = { landmarks: true, // detect hand landmarks or just hand boundary box detector: { modelPath: 'handdetect.json', // hand detector model, can be absolute path or relative to modelBasePath + // can be 'handdetect' or 'handtrack' }, skeleton: { modelPath: 'handskeleton.json', // hand skeleton model, can be absolute path or relative to modelBasePath diff --git a/test/test-main.js b/test/test-main.js index e6645e91..79b43fa2 100644 --- a/test/test-main.js +++ b/test/test-main.js @@ -170,7 +170,7 @@ async function test(Human, inputConfig) { await human.load(); const models = Object.keys(human.models).map((model) => ({ name: model, loaded: (human.models[model] !== null) })); const loaded = models.filter((model) => model.loaded); - if (models.length === 14 && loaded.length === 7) log('state', 'passed: models loaded', models.length, loaded.length); + if (models.length === 15 && loaded.length === 7) log('state', 'passed: models loaded', models.length, loaded.length); else log('error', 'failed: models loaded', models.length, loaded.length); // test warmup sequences @@ -338,8 +338,8 @@ async function test(Human, inputConfig) { // test segmentation res = await human.segmentation(inputCanvas, inputCanvas); - if (!res || !res.width || !res.height) log('error', 'failed: segmentation', res); - else log('state', 'passed: segmentation', [res.width, res.height]); + if (!res || !res.data) log('error', 'failed: segmentation', res); + else log('state', 'passed: segmentation', [res.data.length]); human.env.Canvas = undefined; // tests end diff --git a/wiki b/wiki index d293f4a2..a0497b6d 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit d293f4a20b640e6bc8485dc0f8a2c2147ce33073 +Subproject commit a0497b6d14059099b2764b8f70390f4b6af8db9f