diff --git a/CHANGELOG.md b/CHANGELOG.md index b0f1891b..bc8adb59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # @vladmandic/human Version: **1.4.1** -Description: **Human: AI-powered 3D Face Detection, Face Description & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition** +Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Author: **Vladimir Mandic ** License: **MIT** @@ -9,8 +9,9 @@ Repository: **** ## Changelog -### **HEAD -> main** 2021/04/09 mandic00@live.com +### **HEAD -> main** 2021/04/10 mandic00@live.com +- fix typedoc - exception handling ### **1.4.1** 2021/04/09 mandic00@live.com diff --git a/demo/facematch.js b/demo/facematch.js index c1233c5c..8d81b767 100644 --- a/demo/facematch.js +++ b/demo/facematch.js @@ -1,3 +1,5 @@ +// @ts-nocheck + import Human from '../dist/human.esm.js'; const userConfig = { diff --git a/demo/index.js b/demo/index.js index 51c9e7d6..13f2cc1f 100644 --- a/demo/index.js +++ b/demo/index.js @@ -1,3 +1,5 @@ +// @ts-nocheck + /* global tf */ import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human // import Human from '../dist/human.esm-nobundle.js'; // this requires that tf is loaded manually and bundled before human can be used diff --git a/package.json b/package.json index 2552c283..8969bb0b 100644 --- a/package.json +++ b/package.json @@ -67,13 +67,13 @@ "@vladmandic/pilogger": "^0.2.16", "chokidar": "^3.5.1", "dayjs": "^1.10.4", - "esbuild": "^0.11.6", - "eslint": "^7.23.0", + "esbuild": "^0.11.9", + "eslint": "^7.24.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-plugin-import": "^2.22.1", "eslint-plugin-json": "^2.1.2", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^4.3.1", + "eslint-plugin-promise": "^5.1.0", "rimraf": "^3.0.2", "seedrandom": "^3.0.5", "simple-git": "^2.37.0", diff --git a/src/age/age.ts b/src/age/age.ts index dde92dfb..b7ff470f 100644 --- a/src/age/age.ts +++ b/src/age/age.ts @@ -11,7 +11,7 @@ export async function load(config) { model = await tf.loadGraphModel(join(config.modelBasePath, config.face.age.modelPath)); if (!model || !model.modelUrl) log('load model failed:', config.face.age.modelPath); else if (config.debug) log('load model:', model.modelUrl); - } + } else if (config.debug) log('cached model:', model.modelUrl); return model; } diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index c0c75437..dad57777 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -13,7 +13,7 @@ export async function load(config) { model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath)); if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath); else if (config.debug) log('load model:', model.modelUrl); - } + } else if (config.debug) log('cached model:', model.modelUrl); return model; } diff --git a/src/emotion/emotion.ts b/src/emotion/emotion.ts index 37b7707c..dc52d502 100644 --- a/src/emotion/emotion.ts +++ b/src/emotion/emotion.ts @@ -15,7 +15,7 @@ export async function load(config) { model = await tf.loadGraphModel(join(config.modelBasePath, config.face.emotion.modelPath)); if (!model || !model.modelUrl) log('load model failed:', config.face.emotion.modelPath); else if (config.debug) log('load model:', model.modelUrl); - } + } else if (config.debug) log('cached model:', model.modelUrl); return model; } diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index f29d3cff..645b5582 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -14,7 +14,7 @@ export async function load(config) { model = await tf.loadGraphModel(join(config.modelBasePath, config.face.description.modelPath)); if (!model || !model.modelUrl) log('load model failed:', config.face.description.modelPath); else if (config.debug) log('load model:', model.modelUrl); - } + } else if (config.debug) log('cached model:', model.modelUrl); return model; } diff --git a/src/gender/gender.ts b/src/gender/gender.ts index d622b1aa..511055d5 100644 --- a/src/gender/gender.ts +++ b/src/gender/gender.ts @@ -16,7 +16,7 @@ export async function load(config) { alternative = model.inputs[0].shape[3] === 1; if (!model || !model.modelUrl) log('load model failed:', config.face.gender.modelPath); else if (config.debug) log('load model:', model.modelUrl); - } + } else if (config.debug) log('cached model:', model.modelUrl); return model; } diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index b2c25cc1..6235bab8 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -53,20 +53,26 @@ export class HandPose { } } +let handDetectorModel; +let handPoseModel; export async function load(config): Promise { - const [handDetectorModel, handPoseModel] = await Promise.all([ - config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null, - config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null, - ]); + if (!handDetectorModel || !handPoseModel) { + [handDetectorModel, handPoseModel] = await Promise.all([ + config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null, + config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null, + ]); + if (config.hand.enabled) { + if (!handDetectorModel || !handDetectorModel.modelUrl) log('load model failed:', config.hand.detector.modelPath); + else if (config.debug) log('load model:', handDetectorModel.modelUrl); + if (!handPoseModel || !handPoseModel.modelUrl) log('load model failed:', config.hand.skeleton.modelPath); + else if (config.debug) log('load model:', handPoseModel.modelUrl); + } + } else { + if (config.debug) log('cached model:', handDetectorModel.modelUrl); + if (config.debug) log('cached model:', handPoseModel.modelUrl); + } const handDetector = new handdetector.HandDetector(handDetectorModel, handDetectorModel?.inputs[0].shape[2], anchors.anchors); const handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel, handPoseModel?.inputs[0].shape[2]); const handPose = new HandPose(handPipeline); - - if (config.hand.enabled) { - if (!handDetectorModel || !handDetectorModel.modelUrl) log('load model failed:', config.hand.detector.modelPath); - else if (config.debug) log('load model:', handDetectorModel.modelUrl); - if (!handPoseModel || !handPoseModel.modelUrl) log('load model failed:', config.hand.skeleton.modelPath); - else if (config.debug) log('load model:', handPoseModel.modelUrl); - } return handPose; } diff --git a/src/human.ts b/src/human.ts index dda6eae4..538b4f9c 100644 --- a/src/human.ts +++ b/src/human.ts @@ -343,6 +343,14 @@ export class Human { if (this.config.scoped) this.tf.engine().startScope(); this.analyze('Start Scope:'); + // disable video optimization for inputs of type image + let previousVideoOptimized; + if (input && this.config.videoOptimized && (input instanceof HTMLImageElement || input instanceof Image || input instanceof ImageData || input instanceof ImageBitmap || input instanceof tf.Tensor)) { + log('disabling video optimization'); + previousVideoOptimized = this.config.videoOptimized; + this.config.videoOptimized = false; + } + timeStamp = now(); const process = image.process(input, this.config); if (!process || !process.tensor) { @@ -427,6 +435,7 @@ export class Human { if (this.config.scoped) this.tf.engine().endScope(); this.analyze('End Scope:'); + // run gesture analysis last let gestureRes: any[] = []; if (this.config.gesture.enabled) { timeStamp = now(); @@ -435,6 +444,9 @@ export class Human { else if (this.perf.gesture) delete this.perf.gesture; } + // restore video optimizations if previously disabled + if (previousVideoOptimized) this.config.videoOptimized = previousVideoOptimized; + this.perf.total = Math.trunc(now() - timeStart); this.state = 'idle'; const result = { diff --git a/src/image/image.ts b/src/image/image.ts index 58bd5bcc..2d597c02 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -28,7 +28,8 @@ export function process(input, config): { tensor: typeof tf.Tensor | null, canva throw new Error('Human: Input type is not recognized'); } if (input instanceof tf.Tensor) { - tensor = tf.clone(input); + if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3) tensor = tf.clone(input); + else throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`); } else { const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0)); const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0)); diff --git a/src/nanodet/nanodet.ts b/src/nanodet/nanodet.ts index 60272af9..b2910ff8 100644 --- a/src/nanodet/nanodet.ts +++ b/src/nanodet/nanodet.ts @@ -17,7 +17,7 @@ export async function load(config) { if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`); if (!model || !model.modelUrl) log('load model failed:', config.object.modelPath); else if (config.debug) log('load model:', model.modelUrl); - } + } else if (config.debug) log('cached model:', model.modelUrl); return model; } diff --git a/src/result.ts b/src/result.ts index d6fe69be..90625b5a 100644 --- a/src/result.ts +++ b/src/result.ts @@ -40,7 +40,10 @@ export interface Result { emotion: Array<{ score: number, emotion: string }>, embedding: Array, iris: number, - angle: { roll: number, yaw: number, pitch: number }, + rotation: { + angle: { roll: number, yaw: number, pitch: number }, + matrix: Array<[number, number, number, number, number, number, number, number, number]> + } }>, /** Body results * diff --git a/test/test-node.js b/test/test-node.js index 958eaccc..642df098 100644 --- a/test/test-node.js +++ b/test/test-node.js @@ -30,11 +30,16 @@ const config = { object: { enabled: true }, }; -async function test() { - const human = new Human(config); +async function testInstance(human) { if (human) log.state('passed: create human'); else log.error('failed: create human'); + // if (!human.tf) human.tf = tf; + log.info('human version:', human.version); + log.info('tfjs version:', human.tf.version_core); + log.info('platform:', human.sysinfo.platform); + log.info('agent:', human.sysinfo.agent); + await human.load(); if (human.models) { log.state('passed: load models'); @@ -53,6 +58,28 @@ async function test() { } else { log.error('failed: warmup'); } + const random = tf.randomNormal([1, 1024, 1024, 3]); + const detect = await human.detect(random); + tf.dispose(random); + if (detect) { + log.state('passed: detect:', 'random'); + log.data(' result: face:', detect.face.length, 'body:', detect.body.length, 'hand:', detect.hand.length, 'gesture:', detect.gesture.length, 'object:', detect.object.length); + log.data(' result: performance:', 'load:', detect.performance.load, 'total:', detect.performance.total); + } else { + log.error('failed: detect'); + } +} + +async function test() { + log.info('testing instance#1'); + config.warmup = 'face'; + const human1 = new Human(config); + await testInstance(human1); + + log.info('testing instance#2'); + config.warmup = 'body'; + const human2 = new Human(config); + await testInstance(human2); } test();