added support for multiple instances of human

CHANGELOG.md

@@ -1,7 +1,7 @@
 # @vladmandic/human  
 
 Version: **1.4.1**  
-Description: **Human: AI-powered 3D Face Detection, Face Description & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition**  
+Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**  
 
 Author: **Vladimir Mandic <mandic00@live.com>**  
 License: **MIT** </LICENSE>  
@@ -9,8 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
-### **HEAD -> main** 2021/04/09 mandic00@live.com
+### **HEAD -> main** 2021/04/10 mandic00@live.com
 
+- fix typedoc
 - exception handling
 
 ### **1.4.1** 2021/04/09 mandic00@live.com

demo/face3d.js

@@ -1,3 +1,5 @@
+// @ts-nocheck
+
 import { DoubleSide, Mesh, MeshBasicMaterial, OrthographicCamera, Scene, sRGBEncoding, VideoTexture, WebGLRenderer, BufferGeometry, BufferAttribute } from './helpers/three.js';
 import { OrbitControls } from './helpers/three-orbitControls.js';
 import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human
@@ -7,7 +9,7 @@ const userConfig = {
   async: false,
   profile: false,
   warmup: 'full',
-  videoOptimized: false,
+  videoOptimized: true,
   filter: { enabled: false },
   face: { enabled: true,
     detector: { rotation: false, maxFaces: 1 },

demo/facematch.js

@@ -1,3 +1,5 @@
+// @ts-nocheck
+
 import Human from '../dist/human.esm.js';
 
 const userConfig = {

demo/index.js

@@ -1,3 +1,5 @@
+// @ts-nocheck
+
 /* global tf */
 import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human
 // import Human from '../dist/human.esm-nobundle.js'; // this requires that tf is loaded manually and bundled before human can be used

package.json

@@ -67,13 +67,13 @@
     "@vladmandic/pilogger": "^0.2.16",
     "chokidar": "^3.5.1",
     "dayjs": "^1.10.4",
-    "esbuild": "^0.11.6",
+    "esbuild": "^0.11.9",
-    "eslint": "^7.23.0",
+    "eslint": "^7.24.0",
     "eslint-config-airbnb-base": "^14.2.1",
     "eslint-plugin-import": "^2.22.1",
     "eslint-plugin-json": "^2.1.2",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^4.3.1",
+    "eslint-plugin-promise": "^5.1.0",
     "rimraf": "^3.0.2",
     "seedrandom": "^3.0.5",
     "simple-git": "^2.37.0",

src/age/age.ts

@@ -11,7 +11,7 @@ export async function load(config) {
     model = await tf.loadGraphModel(join(config.modelBasePath, config.face.age.modelPath));
     if (!model || !model.modelUrl) log('load model failed:', config.face.age.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/blazeface/facemesh.ts

@@ -60,20 +60,26 @@ export class MediaPipeFaceMesh {
 
 let faceModels:[any, any, any] = [null, null, null];
 export async function load(config): Promise<MediaPipeFaceMesh> {
-  faceModels = await Promise.all([
+  if ((!faceModels[0] && config.face.enabled) || (!faceModels[1] && config.face.mesh.enabled) || (!faceModels[2] && config.face.iris.enabled)) {
-    (!faceModels[0] && config.face.enabled) ? blazeface.load(config) : null,
+    faceModels = await Promise.all([
-    (!faceModels[1] && config.face.mesh.enabled) ? tf.loadGraphModel(join(config.modelBasePath, config.face.mesh.modelPath), { fromTFHub: config.face.mesh.modelPath.includes('tfhub.dev') }) : null,
+      (!faceModels[0] && config.face.enabled) ? blazeface.load(config) : null,
-    (!faceModels[2] && config.face.iris.enabled) ? tf.loadGraphModel(join(config.modelBasePath, config.face.iris.modelPath), { fromTFHub: config.face.iris.modelPath.includes('tfhub.dev') }) : null,
+      (!faceModels[1] && config.face.mesh.enabled) ? tf.loadGraphModel(join(config.modelBasePath, config.face.mesh.modelPath), { fromTFHub: config.face.mesh.modelPath.includes('tfhub.dev') }) : null,
-  ]);
+      (!faceModels[2] && config.face.iris.enabled) ? tf.loadGraphModel(join(config.modelBasePath, config.face.iris.modelPath), { fromTFHub: config.face.iris.modelPath.includes('tfhub.dev') }) : null,
+    ]);
+    if (config.face.mesh.enabled) {
+      if (!faceModels[1] || !faceModels[1].modelUrl) log('load model failed:', config.face.mesh.modelPath);
+      else if (config.debug) log('load model:', faceModels[1].modelUrl);
+    }
+    if (config.face.iris.enabled) {
+      if (!faceModels[2] || !faceModels[1].modelUrl) log('load model failed:', config.face.iris.modelPath);
+      else if (config.debug) log('load model:', faceModels[2].modelUrl);
+    }
+  } else if (config.debug) {
+    log('cached model:', faceModels[0].model.modelUrl);
+    log('cached model:', faceModels[1].modelUrl);
+    log('cached model:', faceModels[2].modelUrl);
+  }
   const faceMesh = new MediaPipeFaceMesh(faceModels[0], faceModels[1], faceModels[2], config);
-  if (config.face.mesh.enabled) {
-    if (!faceModels[1] || !faceModels[1].modelUrl) log('load model failed:', config.face.age.modelPath);
-    else if (config.debug) log('load model:', faceModels[1].modelUrl);
-  }
-  if (config.face.iris.enabled) {
-    if (!faceModels[2] || !faceModels[1].modelUrl) log('load model failed:', config.face.age.modelPath);
-    else if (config.debug) log('load model:', faceModels[2].modelUrl);
-  }
   return faceMesh;
 }
 

src/blazepose/blazepose.ts

@@ -12,7 +12,7 @@ export async function load(config) {
     model.height = parseInt(model.signature.inputs['input_1:0'].tensorShape.dim[1].size);
     if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/efficientpose/efficientpose.ts

@@ -13,7 +13,7 @@ export async function load(config) {
     model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath));
     if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/embedding/embedding.ts

@@ -11,7 +11,7 @@ export async function load(config) {
     model = await tf.loadGraphModel(join(config.modelBasePath, config.face.embedding.modelPath));
     if (!model || !model.modelUrl) log('load model failed:', config.face.embedding.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/emotion/emotion.ts

@@ -15,7 +15,7 @@ export async function load(config) {
     model = await tf.loadGraphModel(join(config.modelBasePath, config.face.emotion.modelPath));
     if (!model || !model.modelUrl) log('load model failed:', config.face.emotion.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/faceres/faceres.ts

@@ -14,7 +14,7 @@ export async function load(config) {
     model = await tf.loadGraphModel(join(config.modelBasePath, config.face.description.modelPath));
     if (!model || !model.modelUrl) log('load model failed:', config.face.description.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/gender/gender.ts

@@ -16,7 +16,7 @@ export async function load(config) {
     alternative = model.inputs[0].shape[3] === 1;
     if (!model || !model.modelUrl) log('load model failed:', config.face.gender.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/handpose/handpose.ts

@@ -53,20 +53,26 @@ export class HandPose {
   }
 }
 
+let handDetectorModel;
+let handPoseModel;
 export async function load(config): Promise<HandPose> {
-  const [handDetectorModel, handPoseModel] = await Promise.all([
+  if (!handDetectorModel || !handPoseModel) {
-    config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
+    [handDetectorModel, handPoseModel] = await Promise.all([
-    config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,
+      config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
-  ]);
+      config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,
+    ]);
+    if (config.hand.enabled) {
+      if (!handDetectorModel || !handDetectorModel.modelUrl) log('load model failed:', config.hand.detector.modelPath);
+      else if (config.debug) log('load model:', handDetectorModel.modelUrl);
+      if (!handPoseModel || !handPoseModel.modelUrl) log('load model failed:', config.hand.skeleton.modelPath);
+      else if (config.debug) log('load model:', handPoseModel.modelUrl);
+    }
+  } else {
+    if (config.debug) log('cached model:', handDetectorModel.modelUrl);
+    if (config.debug) log('cached model:', handPoseModel.modelUrl);
+  }
   const handDetector = new handdetector.HandDetector(handDetectorModel, handDetectorModel?.inputs[0].shape[2], anchors.anchors);
   const handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel, handPoseModel?.inputs[0].shape[2]);
   const handPose = new HandPose(handPipeline);
-
-  if (config.hand.enabled) {
-    if (!handDetectorModel || !handDetectorModel.modelUrl) log('load model failed:', config.hand.detector.modelPath);
-    else if (config.debug) log('load model:', handDetectorModel.modelUrl);
-    if (!handPoseModel || !handPoseModel.modelUrl) log('load model failed:', config.hand.skeleton.modelPath);
-    else if (config.debug) log('load model:', handPoseModel.modelUrl);
-  }
   return handPose;
 }

src/human.ts

@@ -343,6 +343,14 @@ export class Human {
       if (this.config.scoped) this.tf.engine().startScope();
       this.analyze('Start Scope:');
 
+      // disable video optimization for inputs of type image
+      let previousVideoOptimized;
+      if (input && this.config.videoOptimized && (input instanceof HTMLImageElement || input instanceof Image || input instanceof ImageData || input instanceof ImageBitmap || input instanceof tf.Tensor)) {
+        log('disabling video optimization');
+        previousVideoOptimized = this.config.videoOptimized;
+        this.config.videoOptimized = false;
+      }
+
       timeStamp = now();
       const process = image.process(input, this.config);
       if (!process || !process.tensor) {
@@ -427,6 +435,7 @@ export class Human {
       if (this.config.scoped) this.tf.engine().endScope();
       this.analyze('End Scope:');
 
+      // run gesture analysis last
       let gestureRes: any[] = [];
       if (this.config.gesture.enabled) {
         timeStamp = now();
@@ -435,6 +444,9 @@ export class Human {
         else if (this.perf.gesture) delete this.perf.gesture;
       }
 
+      // restore video optimizations if previously disabled
+      if (previousVideoOptimized) this.config.videoOptimized = previousVideoOptimized;
+
       this.perf.total = Math.trunc(now() - timeStart);
       this.state = 'idle';
       const result = {

src/image/image.ts

@@ -28,7 +28,8 @@ export function process(input, config): { tensor: typeof tf.Tensor | null, canva
     throw new Error('Human: Input type is not recognized');
   }
   if (input instanceof tf.Tensor) {
-    tensor = tf.clone(input);
+    if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3) tensor = tf.clone(input);
+    else throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`);
   } else {
     const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0));
     const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0));

src/nanodet/nanodet.ts

@@ -17,7 +17,7 @@ export async function load(config) {
     if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`);
     if (!model || !model.modelUrl) log('load model failed:', config.object.modelPath);
     else if (config.debug) log('load model:', model.modelUrl);
-  }
+  } else if (config.debug) log('cached model:', model.modelUrl);
   return model;
 }
 

src/posenet/posenet.ts

@@ -5,6 +5,8 @@ import * as decodeMultiple from './decodeMultiple';
 import * as decodePose from './decodePose';
 import * as util from './util';
 
+let model;
+
 async function estimateMultiple(input, res, config, inputSize) {
   return new Promise(async (resolve) => {
     const allTensorBuffers = await util.toTensorBuffers3D([res.heatmapScores, res.offsets, res.displacementFwd, res.displacementBwd]);
@@ -29,9 +31,9 @@ async function estimateSingle(input, res, config, inputSize) {
 export class PoseNet {
   baseModel: any;
   inputSize: number
-  constructor(model) {
+  constructor(mobilenet) {
-    this.baseModel = model;
+    this.baseModel = mobilenet;
-    this.inputSize = model.model.inputs[0].shape[1];
+    this.inputSize = mobilenet.model.inputs[0].shape[1];
     if (this.inputSize < 128) this.inputSize = 257;
   }
 
@@ -58,9 +60,12 @@ export class PoseNet {
 }
 
 export async function load(config) {
-  const model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath));
+  if (!model) {
+    model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath));
+    if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath);
+    else if (config.debug) log('load model:', model.modelUrl);
+  } else if (config.debug) log('cached model:', model.modelUrl);
   const mobilenet = new modelBase.BaseModel(model);
-  if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath);
+  const poseNet = new PoseNet(mobilenet);
-  else if (config.debug) log('load model:', model.modelUrl);
+  return poseNet;
-  return new PoseNet(mobilenet);
 }

src/result.ts

@@ -40,7 +40,10 @@ export interface Result {
     emotion: Array<{ score: number, emotion: string }>,
     embedding: Array<number>,
     iris: number,
-    angle: { roll: number, yaw: number, pitch: number },
+    rotation: {
+      angle: { roll: number, yaw: number, pitch: number },
+      matrix: Array<[number, number, number, number, number, number, number, number, number]>
+    }
   }>,
   /** Body results
    *

test/test-node.js

@@ -30,11 +30,16 @@ const config = {
   object: { enabled: true },
 };
 
-async function test() {
+async function testInstance(human) {
-  const human = new Human(config);
   if (human) log.state('passed: create human');
   else log.error('failed: create human');
 
+  // if (!human.tf) human.tf = tf;
+  log.info('human version:', human.version);
+  log.info('tfjs version:', human.tf.version_core);
+  log.info('platform:', human.sysinfo.platform);
+  log.info('agent:', human.sysinfo.agent);
+
   await human.load();
   if (human.models) {
     log.state('passed: load models');
@@ -53,6 +58,28 @@ async function test() {
   } else {
     log.error('failed: warmup');
   }
+  const random = tf.randomNormal([1, 1024, 1024, 3]);
+  const detect = await human.detect(random);
+  tf.dispose(random);
+  if (detect) {
+    log.state('passed: detect:', 'random');
+    log.data(' result: face:', detect.face.length, 'body:', detect.body.length, 'hand:', detect.hand.length, 'gesture:', detect.gesture.length, 'object:', detect.object.length);
+    log.data(' result: performance:', 'load:', detect.performance.load, 'total:', detect.performance.total);
+  } else {
+    log.error('failed: detect');
+  }
+}
+
+async function test() {
+  log.info('testing instance#1');
+  config.warmup = 'face';
+  const human1 = new Human(config);
+  await testInstance(human1);
+
+  log.info('testing instance#2');
+  config.warmup = 'body';
+  const human2 = new Human(config);
+  await testInstance(human2);
 }
 
 test();

types/posenet/posenet.d.ts

@@ -1,7 +1,7 @@
 export declare class PoseNet {
     baseModel: any;
     inputSize: number;
-    constructor(model: any);
+    constructor(mobilenet: any);
     estimatePoses(input: any, config: any): Promise<unknown>;
     dispose(): void;
 }

types/result.d.ts

@@ -46,10 +46,13 @@ export interface Result {
         }>;
         embedding: Array<number>;
         iris: number;
-        angle: {
+        rotation: {
-            roll: number;
+            angle: {
-            yaw: number;
+                roll: number;
-            pitch: number;
+                yaw: number;
+                pitch: number;
+            };
+            matrix: Array<[number, number, number, number, number, number, number, number, number]>;
         };
     }>;
     /** Body results