added support for multiple instances of human

CHANGELOG.md

@@ -1,7 +1,7 @@
 # @vladmandic/human  
 
 Version: **1.4.1**  
-Description: **Human: AI-powered 3D Face Detection, Face Description & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition**  
+Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**  
 
 Author: **Vladimir Mandic <mandic00@live.com>**  
 License: **MIT** </LICENSE>  
@@ -9,8 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
-### **HEAD -> main** 2021/04/09 mandic00@live.com
+### **HEAD -> main** 2021/04/10 mandic00@live.com
 
+- fix typedoc
 - exception handling
 
 ### **1.4.1** 2021/04/09 mandic00@live.com

demo/facematch.js

@@ -1,3 +1,5 @@
+// @ts-nocheck
+
 import Human from '../dist/human.esm.js';
 
 const userConfig = {

demo/index.js

@@ -1,3 +1,5 @@
+// @ts-nocheck
+
 /* global tf */
 import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human
 // import Human from '../dist/human.esm-nobundle.js'; // this requires that tf is loaded manually and bundled before human can be used

package.json

@@ -67,13 +67,13 @@
     "@vladmandic/pilogger": "^0.2.16",
     "chokidar": "^3.5.1",
     "dayjs": "^1.10.4",
-    "esbuild": "^0.11.6",
+    "esbuild": "^0.11.9",
-    "eslint": "^7.23.0",
+    "eslint": "^7.24.0",
     "eslint-config-airbnb-base": "^14.2.1",
     "eslint-plugin-import": "^2.22.1",
     "eslint-plugin-json": "^2.1.2",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^4.3.1",
+    "eslint-plugin-promise": "^5.1.0",
     "rimraf": "^3.0.2",
     "seedrandom": "^3.0.5",
     "simple-git": "^2.37.0",

src/age/age.ts

@@ -11,7 +11,7 @@ export async function load(config) {
     model = await tf.loadGraphModel(join(config.modelBasePath, config.face.age.modelPath));
     if (!model || !model.modelUrl) log('load model failed:', config.face.age.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/efficientpose/efficientpose.ts

@@ -13,7 +13,7 @@ export async function load(config) {
     model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath));
     if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/emotion/emotion.ts

@@ -15,7 +15,7 @@ export async function load(config) {
     model = await tf.loadGraphModel(join(config.modelBasePath, config.face.emotion.modelPath));
     if (!model || !model.modelUrl) log('load model failed:', config.face.emotion.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/faceres/faceres.ts

@@ -14,7 +14,7 @@ export async function load(config) {
     model = await tf.loadGraphModel(join(config.modelBasePath, config.face.description.modelPath));
     if (!model || !model.modelUrl) log('load model failed:', config.face.description.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/gender/gender.ts

@@ -16,7 +16,7 @@ export async function load(config) {
     alternative = model.inputs[0].shape[3] === 1;
     if (!model || !model.modelUrl) log('load model failed:', config.face.gender.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/handpose/handpose.ts

@@ -53,20 +53,26 @@ export class HandPose {
   }
 }
 
+let handDetectorModel;
+let handPoseModel;
 export async function load(config): Promise<HandPose> {
-  const [handDetectorModel, handPoseModel] = await Promise.all([
+  if (!handDetectorModel || !handPoseModel) {
-    config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
+    [handDetectorModel, handPoseModel] = await Promise.all([
-    config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,
+      config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
-  ]);
+      config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,
+    ]);
+    if (config.hand.enabled) {
+      if (!handDetectorModel || !handDetectorModel.modelUrl) log('load model failed:', config.hand.detector.modelPath);
+      else if (config.debug) log('load model:', handDetectorModel.modelUrl);
+      if (!handPoseModel || !handPoseModel.modelUrl) log('load model failed:', config.hand.skeleton.modelPath);
+      else if (config.debug) log('load model:', handPoseModel.modelUrl);
+    }
+  } else {
+    if (config.debug) log('cached model:', handDetectorModel.modelUrl);
+    if (config.debug) log('cached model:', handPoseModel.modelUrl);
+  }
   const handDetector = new handdetector.HandDetector(handDetectorModel, handDetectorModel?.inputs[0].shape[2], anchors.anchors);
   const handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel, handPoseModel?.inputs[0].shape[2]);
   const handPose = new HandPose(handPipeline);
-
-  if (config.hand.enabled) {
-    if (!handDetectorModel || !handDetectorModel.modelUrl) log('load model failed:', config.hand.detector.modelPath);
-    else if (config.debug) log('load model:', handDetectorModel.modelUrl);
-    if (!handPoseModel || !handPoseModel.modelUrl) log('load model failed:', config.hand.skeleton.modelPath);
-    else if (config.debug) log('load model:', handPoseModel.modelUrl);
-  }
   return handPose;
 }

src/human.ts

@@ -343,6 +343,14 @@ export class Human {
       if (this.config.scoped) this.tf.engine().startScope();
       this.analyze('Start Scope:');
 
+      // disable video optimization for inputs of type image
+      let previousVideoOptimized;
+      if (input && this.config.videoOptimized && (input instanceof HTMLImageElement || input instanceof Image || input instanceof ImageData || input instanceof ImageBitmap || input instanceof tf.Tensor)) {
+        log('disabling video optimization');
+        previousVideoOptimized = this.config.videoOptimized;
+        this.config.videoOptimized = false;
+      }
+
       timeStamp = now();
       const process = image.process(input, this.config);
       if (!process || !process.tensor) {
@@ -427,6 +435,7 @@ export class Human {
       if (this.config.scoped) this.tf.engine().endScope();
       this.analyze('End Scope:');
 
+      // run gesture analysis last
       let gestureRes: any[] = [];
       if (this.config.gesture.enabled) {
         timeStamp = now();
@@ -435,6 +444,9 @@ export class Human {
         else if (this.perf.gesture) delete this.perf.gesture;
       }
 
+      // restore video optimizations if previously disabled
+      if (previousVideoOptimized) this.config.videoOptimized = previousVideoOptimized;
+
       this.perf.total = Math.trunc(now() - timeStart);
       this.state = 'idle';
       const result = {

src/image/image.ts

@@ -28,7 +28,8 @@ export function process(input, config): { tensor: typeof tf.Tensor | null, canva
     throw new Error('Human: Input type is not recognized');
   }
   if (input instanceof tf.Tensor) {
-    tensor = tf.clone(input);
+    if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3) tensor = tf.clone(input);
+    else throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`);
   } else {
     const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0));
     const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0));

src/nanodet/nanodet.ts

@@ -17,7 +17,7 @@ export async function load(config) {
     if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`);
     if (!model || !model.modelUrl) log('load model failed:', config.object.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/result.ts

@@ -40,7 +40,10 @@ export interface Result {
     emotion: Array<{ score: number, emotion: string }>,
     embedding: Array<number>,
     iris: number,
-    angle: { roll: number, yaw: number, pitch: number },
+    rotation: {
+      angle: { roll: number, yaw: number, pitch: number },
+      matrix: Array<[number, number, number, number, number, number, number, number, number]>
+    }
   }>,
   /** Body results
    *

test/test-node.js

@@ -30,11 +30,16 @@ const config = {
   object: { enabled: true },
 };
 
-async function test() {
+async function testInstance(human) {
-  const human = new Human(config);
   if (human) log.state('passed: create human');
   else log.error('failed: create human');
 
+  // if (!human.tf) human.tf = tf;
+  log.info('human version:', human.version);
+  log.info('tfjs version:', human.tf.version_core);
+  log.info('platform:', human.sysinfo.platform);
+  log.info('agent:', human.sysinfo.agent);
+
   await human.load();
   if (human.models) {
     log.state('passed: load models');
@@ -53,6 +58,28 @@ async function test() {
   } else {
     log.error('failed: warmup');
   }
+  const random = tf.randomNormal([1, 1024, 1024, 3]);
+  const detect = await human.detect(random);
+  tf.dispose(random);
+  if (detect) {
+    log.state('passed: detect:', 'random');
+    log.data(' result: face:', detect.face.length, 'body:', detect.body.length, 'hand:', detect.hand.length, 'gesture:', detect.gesture.length, 'object:', detect.object.length);
+    log.data(' result: performance:', 'load:', detect.performance.load, 'total:', detect.performance.total);
+  } else {
+    log.error('failed: detect');
+  }
+}
+
+async function test() {
+  log.info('testing instance#1');
+  config.warmup = 'face';
+  const human1 = new Human(config);
+  await testInstance(human1);
+
+  log.info('testing instance#2');
+  config.warmup = 'body';
+  const human2 = new Human(config);
+  await testInstance(human2);
 }
 
 test();