convert blazeface to module

CHANGELOG.md

@@ -11,10 +11,16 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ### **HEAD -> main** 2021/04/25 mandic00@live.com
 
+- build nodejs deliverables in non-minified form
+- stop building sourcemaps for nodejs deliverables
+- remove deallocate, profile, scoped
+- replaced maxfaces, maxdetections, maxhands, maxresults with maxdetected
+- replaced nmsradius with built-in default
+- unified minconfidence and scorethresdold as minconfidence
+- add exception handlers to all demos
+- remove blazeface-front and add unhandledrejection handler
 - major update for 1.8 release candidate
-
-### **origin/main** 2021/04/25 mandic00@live.com
-
+- enable webworker detection
 
 ### **1.7.1** 2021/04/25 mandic00@live.com
 

TODO.md

@@ -6,8 +6,7 @@ N/A
 
 ## Exploring Features
 
-- Implement built-in input handler for `http:`, `https:`, `file:`
-- Canvas.js for WASM on NodeJS
+N/A
 
 ## Explore Models
 

demo/index.js

@@ -7,10 +7,9 @@ import Menu from './helpers/menu.js';
 import GLBench from './helpers/gl-bench.js';
 import webRTC from './helpers/webrtc.js';
 
-const userConfig = {};
+// const userConfig = {};
 let human;
 
-/*
 const userConfig = {
   backend: 'humangl',
   async: false,
@@ -21,7 +20,7 @@ const userConfig = {
     enabled: false,
     flip: false,
   },
-  face: { enabled: false,
+  face: { enabled: true,
     mesh: { enabled: true },
     iris: { enabled: true },
     description: { enabled: false },
@@ -29,11 +28,10 @@ const userConfig = {
   },
   hand: { enabled: false },
   gesture: { enabled: false },
-  body: { enabled: true, modelPath: 'posenet.json' },
+  body: { enabled: false, modelPath: 'posenet.json' },
   // body: { enabled: true, modelPath: 'blazepose.json' },
   // object: { enabled: true },
 };
-*/
 
 // ui options
 const ui = {

server/build.log

@@ -338,3 +338,20 @@
 2021-04-25 14:28:40 INFO:  Generate types: ["src/human.ts"] 
 2021-04-25 14:28:45 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"] 
 2021-04-25 14:28:45 INFO:  Generate TypeDocs: ["src/human.ts"] 
+2021-04-25 16:54:54 INFO:  @vladmandic/human version 1.8.0 
+2021-04-25 16:54:54 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0 
+2021-04-25 16:54:54 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} 
+2021-04-25 16:54:54 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":696,"outputFiles":"dist/tfjs.esm.js"} 
+2021-04-25 16:54:54 STATE: Build for: node type: node: {"imports":35,"importBytes":414277,"outputBytes":373298,"outputFiles":"dist/human.node.js"} 
+2021-04-25 16:54:54 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":700,"outputFiles":"dist/tfjs.esm.js"} 
+2021-04-25 16:54:54 STATE: Build for: nodeGPU type: node: {"imports":35,"importBytes":414281,"outputBytes":373302,"outputFiles":"dist/human.node-gpu.js"} 
+2021-04-25 16:54:54 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":746,"outputFiles":"dist/tfjs.esm.js"} 
+2021-04-25 16:54:54 STATE: Build for: nodeWASM type: node: {"imports":35,"importBytes":414327,"outputBytes":373352,"outputFiles":"dist/human.node-wasm.js"} 
+2021-04-25 16:54:54 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"} 
+2021-04-25 16:54:54 STATE: Build for: browserNoBundle type: esm: {"imports":35,"importBytes":414975,"outputBytes":229880,"outputFiles":"dist/human.esm-nobundle.js"} 
+2021-04-25 16:54:55 STATE: Build for: browserBundle type: tfjs: {"modules":1267,"moduleBytes":4085087,"imports":7,"importBytes":2488,"outputBytes":1101728,"outputFiles":"dist/tfjs.esm.js"} 
+2021-04-25 16:54:56 STATE: Build for: browserBundle type: iife: {"imports":35,"importBytes":1515309,"outputBytes":1328044,"outputFiles":"dist/human.js"} 
+2021-04-25 16:54:56 STATE: Build for: browserBundle type: esm: {"imports":35,"importBytes":1515309,"outputBytes":1328002,"outputFiles":"dist/human.esm.js"} 
+2021-04-25 16:54:56 INFO:  Generate types: ["src/human.ts"] 
+2021-04-25 16:55:01 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"] 
+2021-04-25 16:55:01 INFO:  Generate TypeDocs: ["src/human.ts"] 

src/blazeface/blazeface.ts

@@ -1,40 +1,9 @@
 import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
+import * as box from './box';
+import * as util from './util';
 
-const NUM_LANDMARKS = 6;
-
-function generateAnchors(inputSize) {
-  const spec = { strides: [inputSize / 16, inputSize / 8], anchors: [2, 6] };
-  const anchors: Array<[number, number]> = [];
-  for (let i = 0; i < spec.strides.length; i++) {
-    const stride = spec.strides[i];
-    const gridRows = Math.floor((inputSize + stride - 1) / stride);
-    const gridCols = Math.floor((inputSize + stride - 1) / stride);
-    const anchorsNum = spec.anchors[i];
-    for (let gridY = 0; gridY < gridRows; gridY++) {
-      const anchorY = stride * (gridY + 0.5);
-      for (let gridX = 0; gridX < gridCols; gridX++) {
-        const anchorX = stride * (gridX + 0.5);
-        for (let n = 0; n < anchorsNum; n++) {
-          anchors.push([anchorX, anchorY]);
-        }
-      }
-    }
-  }
-  return anchors;
-}
-
-export const disposeBox = (box) => {
-  box.startEndTensor.dispose();
-  box.startPoint.dispose();
-  box.endPoint.dispose();
-};
-
-const createBox = (startEndTensor) => ({
-  startEndTensor,
-  startPoint: tf.slice(startEndTensor, [0, 0], [-1, 2]),
-  endPoint: tf.slice(startEndTensor, [0, 2], [-1, 2]),
-});
+const keypointsCount = 6;
 
 function decodeBounds(boxOutputs, anchors, inputSize) {
   const boxStarts = tf.slice(boxOutputs, [0, 1], [-1, 2]);
@@ -60,7 +29,7 @@ export class BlazeFaceModel {
 
   constructor(model, config) {
     this.model = model;
-    this.anchorsData = generateAnchors(model.inputs[0].shape[1]);
+    this.anchorsData = util.generateAnchors(model.inputs[0].shape[1]);
     this.anchors = tf.tensor2d(this.anchorsData);
     this.inputSize = model.inputs[0].shape[2];
     this.config = config;
@@ -106,10 +75,10 @@ export class BlazeFaceModel {
       const boxIndex = boxIndices[i];
       const confidence = scoresVal[boxIndex];
       if (confidence > this.config.face.detector.minConfidence) {
-        const box = createBox(boundingBoxes[i]);
+        const localBox = box.createBox(boundingBoxes[i]);
         const anchor = this.anchorsData[boxIndex];
-        const landmarks = tf.tidy(() => tf.slice(batch, [boxIndex, NUM_LANDMARKS - 1], [1, -1]).squeeze().reshape([NUM_LANDMARKS, -1]));
-        annotatedBoxes.push({ box, landmarks, anchor, confidence });
+        const landmarks = tf.tidy(() => tf.slice(batch, [boxIndex, keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
+        annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence });
       }
     }
     batch.dispose();

src/blazeface/box.ts

@@ -50,3 +50,23 @@ export function squarifyBox(box) {
   const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
   return { startPoint, endPoint, landmarks: box.landmarks };
 }
+
+export function calculateLandmarksBoundingBox(landmarks) {
+  const xs = landmarks.map((d) => d[0]);
+  const ys = landmarks.map((d) => d[1]);
+  const startPoint = [Math.min(...xs), Math.min(...ys)];
+  const endPoint = [Math.max(...xs), Math.max(...ys)];
+  return { startPoint, endPoint, landmarks };
+}
+
+export const disposeBox = (t) => {
+  t.startEndTensor.dispose();
+  t.startPoint.dispose();
+  t.endPoint.dispose();
+};
+
+export const createBox = (startEndTensor) => ({
+  startEndTensor,
+  startPoint: tf.slice(startEndTensor, [0, 0], [-1, 2]),
+  endPoint: tf.slice(startEndTensor, [0, 2], [-1, 2]),
+});

src/blazeface/facemesh.ts

@@ -4,62 +4,54 @@ import * as blazeface from './blazeface';
 import * as facepipeline from './facepipeline';
 import * as coords from './coords';
 
-export class MediaPipeFaceMesh {
-  facePipeline: any;
-  config: any;
+let faceModels:[any, any, any] = [null, null, null];
+let facePipeline;
 
-  constructor(blazeFace, blazeMeshModel, irisModel, config) {
-    this.facePipeline = new facepipeline.Pipeline(blazeFace, blazeMeshModel, irisModel);
-    this.config = config;
-  }
-
-  async estimateFaces(input, config): Promise<{ confidence, boxConfidence, faceConfidence, box, mesh, boxRaw, meshRaw, annotations, image }[]> {
-    const predictions = await this.facePipeline.predict(input, config);
-    const results: Array<{ confidence, boxConfidence, faceConfidence, box, mesh, boxRaw, meshRaw, annotations, image }> = [];
-    for (const prediction of (predictions || [])) {
-      if (prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing
-      const mesh = prediction.coords ? prediction.coords.arraySync() : [];
-      const meshRaw = mesh.map((pt) => [
-        pt[0] / input.shape[2],
-        pt[1] / input.shape[1],
-        pt[2] / this.facePipeline.meshSize,
-      ]);
-      const annotations = {};
-      if (mesh && mesh.length > 0) {
-        for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => mesh[index]);
-      }
-      const box = prediction.box ? [
-        Math.max(0, prediction.box.startPoint[0]),
-        Math.max(0, prediction.box.startPoint[1]),
-        Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
-        Math.min(input.shape[1], prediction.box.endPoint[1]) - Math.max(0, prediction.box.startPoint[1]),
-      ] : 0;
-      const boxRaw = prediction.box ? [
-        prediction.box.startPoint[0] / input.shape[2],
-        prediction.box.startPoint[1] / input.shape[1],
-        (prediction.box.endPoint[0] - prediction.box.startPoint[0]) / input.shape[2],
-        (prediction.box.endPoint[1] - prediction.box.startPoint[1]) / input.shape[1],
-      ] : [];
-      results.push({
-        confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
-        boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
-        faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
-        box,
-        boxRaw,
-        mesh,
-        meshRaw,
-        annotations,
-        image: prediction.image ? prediction.image.clone() : null,
-      });
-      if (prediction.coords) prediction.coords.dispose();
-      if (prediction.image) prediction.image.dispose();
+export async function predict(input, config): Promise<{ confidence, boxConfidence, faceConfidence, box, mesh, boxRaw, meshRaw, annotations, image }[]> {
+  const predictions = await facePipeline.predict(input, config);
+  const results: Array<{ confidence, boxConfidence, faceConfidence, box, mesh, boxRaw, meshRaw, annotations, image }> = [];
+  for (const prediction of (predictions || [])) {
+    if (prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing
+    const mesh = prediction.coords ? prediction.coords.arraySync() : [];
+    const meshRaw = mesh.map((pt) => [
+      pt[0] / input.shape[2],
+      pt[1] / input.shape[1],
+      pt[2] / facePipeline.meshSize,
+    ]);
+    const annotations = {};
+    if (mesh && mesh.length > 0) {
+      for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => mesh[index]);
     }
-    return results;
+    const box = prediction.box ? [
+      Math.max(0, prediction.box.startPoint[0]),
+      Math.max(0, prediction.box.startPoint[1]),
+      Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
+      Math.min(input.shape[1], prediction.box.endPoint[1]) - Math.max(0, prediction.box.startPoint[1]),
+    ] : 0;
+    const boxRaw = prediction.box ? [
+      prediction.box.startPoint[0] / input.shape[2],
+      prediction.box.startPoint[1] / input.shape[1],
+      (prediction.box.endPoint[0] - prediction.box.startPoint[0]) / input.shape[2],
+      (prediction.box.endPoint[1] - prediction.box.startPoint[1]) / input.shape[1],
+    ] : [];
+    results.push({
+      confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
+      boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
+      faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
+      box,
+      boxRaw,
+      mesh,
+      meshRaw,
+      annotations,
+      image: prediction.image ? prediction.image.clone() : null,
+    });
+    if (prediction.coords) prediction.coords.dispose();
+    if (prediction.image) prediction.image.dispose();
   }
+  return results;
 }
 
-let faceModels:[any, any, any] = [null, null, null];
-export async function load(config): Promise<MediaPipeFaceMesh> {
+export async function load(config): Promise<[Object, Object, Object]> {
   if ((!faceModels[0] && config.face.enabled) || (!faceModels[1] && config.face.mesh.enabled) || (!faceModels[2] && config.face.iris.enabled)) {
     faceModels = await Promise.all([
       (!faceModels[0] && config.face.enabled) ? blazeface.load(config) : null,
@@ -79,8 +71,8 @@ export async function load(config): Promise<MediaPipeFaceMesh> {
     log('cached model:', faceModels[1].modelUrl);
     log('cached model:', faceModels[2].modelUrl);
   }
-  const faceMesh = new MediaPipeFaceMesh(faceModels[0], faceModels[1], faceModels[2], config);
-  return faceMesh;
+  facePipeline = new facepipeline.Pipeline(faceModels[0], faceModels[1], faceModels[2]);
+  return faceModels;
 }
 
 export const triangulation = coords.TRI468;

src/blazeface/facepipeline.ts

@@ -1,4 +1,3 @@
-/* eslint-disable class-methods-use-this */
 import * as tf from '../../dist/tfjs.esm.js';
 import * as bounding from './box';
 import * as util from './util';
@@ -97,6 +96,7 @@ export class Pipeline {
     ]));
   }
 
+  // eslint-disable-next-line class-methods-use-this
   getLeftToRightEyeDepthDifference(rawCoords) {
     const leftEyeZ = rawCoords[eyeLandmarks.leftBounds[0]][2];
     const rightEyeZ = rawCoords[eyeLandmarks.rightBounds[0]][2];
@@ -105,7 +105,7 @@ export class Pipeline {
 
   // Returns a box describing a cropped region around the eye fit for passing to the iris model.
   getEyeBox(rawCoords, face, eyeInnerCornerIndex, eyeOuterCornerIndex, flip = false) {
-    const box = bounding.squarifyBox(bounding.enlargeBox(this.calculateLandmarksBoundingBox([rawCoords[eyeInnerCornerIndex], rawCoords[eyeOuterCornerIndex]]), this.irisEnlarge));
+    const box = bounding.squarifyBox(bounding.enlargeBox(bounding.calculateLandmarksBoundingBox([rawCoords[eyeInnerCornerIndex], rawCoords[eyeOuterCornerIndex]]), this.irisEnlarge));
     const boxSize = bounding.getBoxSize(box);
     let crop = tf.image.cropAndResize(face, [[
       box.startPoint[1] / this.meshSize,
@@ -134,6 +134,7 @@ export class Pipeline {
   }
 
   // The z-coordinates returned for the iris are unreliable, so we take the z values from the surrounding keypoints.
+  // eslint-disable-next-line class-methods-use-this
   getAdjustedIrisCoords(rawCoords, irisCoords, direction) {
     const upperCenterZ = rawCoords[coords.MESH_ANNOTATIONS[`${direction}EyeUpper0`][irisLandmarks.upperCenter]][2];
     const lowerCenterZ = rawCoords[coords.MESH_ANNOTATIONS[`${direction}EyeLower0`][irisLandmarks.lowerCenter]][2];
@@ -265,7 +266,7 @@ export class Pipeline {
 
       // override box from detection with one calculated from mesh
       const transformedCoordsData = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
-      box = bounding.enlargeBox(this.calculateLandmarksBoundingBox(transformedCoordsData), 1.5); // redefine box with mesh calculated one
+      box = bounding.enlargeBox(bounding.calculateLandmarksBoundingBox(transformedCoordsData), 1.5); // redefine box with mesh calculated one
       const transformedCoords = tf.tensor2d(transformedCoordsData);
 
       // do rotation one more time with mesh keypoints if we want to return perfect image
@@ -302,12 +303,4 @@ export class Pipeline {
 
     return results;
   }
-
-  calculateLandmarksBoundingBox(landmarks) {
-    const xs = landmarks.map((d) => d[0]);
-    const ys = landmarks.map((d) => d[1]);
-    const startPoint = [Math.min(...xs), Math.min(...ys)];
-    const endPoint = [Math.max(...xs), Math.max(...ys)];
-    return { startPoint, endPoint, landmarks };
-  }
 }

src/blazeface/util.ts

@@ -87,3 +87,24 @@ export function rotatePoint(homogeneousCoordinate, rotationMatrix) {
 export function xyDistanceBetweenPoints(a, b) {
   return Math.sqrt(((a[0] - b[0]) ** 2) + ((a[1] - b[1]) ** 2));
 }
+
+export function generateAnchors(inputSize) {
+  const spec = { strides: [inputSize / 16, inputSize / 8], anchors: [2, 6] };
+  const anchors: Array<[number, number]> = [];
+  for (let i = 0; i < spec.strides.length; i++) {
+    const stride = spec.strides[i];
+    const gridRows = Math.floor((inputSize + stride - 1) / stride);
+    const gridCols = Math.floor((inputSize + stride - 1) / stride);
+    const anchorsNum = spec.anchors[i];
+    for (let gridY = 0; gridY < gridRows; gridY++) {
+      const anchorY = stride * (gridY + 0.5);
+      for (let gridX = 0; gridX < gridCols; gridX++) {
+        const anchorX = stride * (gridX + 0.5);
+        for (let n = 0; n < anchorsNum; n++) {
+          anchors.push([anchorX, anchorY]);
+        }
+      }
+    }
+  }
+  return anchors;
+}

src/faceall.ts

@@ -1,5 +1,6 @@
 import { log, now } from './helpers';
 import * as tf from '../dist/tfjs.esm.js';
+import * as facemesh from './blazeface/facemesh';
 import * as emotion from './emotion/emotion';
 import * as faceres from './faceres/faceres';
 
@@ -130,7 +131,7 @@ export const detectFace = async (parent, input): Promise<any> => {
     }> = [];
   parent.state = 'run:face';
   timeStamp = now();
-  const faces = await parent.models.face?.estimateFaces(input, parent.config);
+  const faces = await facemesh.predict(input, parent.config);
   parent.perf.face = Math.trunc(now() - timeStamp);
   if (!faces) return [];
   for (const face of faces) {

src/handpose/handdetector.ts

@@ -1,5 +1,6 @@
 import * as tf from '../../dist/tfjs.esm.js';
 import * as box from './box';
+import * as anchors from './anchors';
 
 export class HandDetector {
   model: any;
@@ -9,13 +10,13 @@ export class HandDetector {
   inputSizeTensor: any;
   doubleInputSizeTensor: any;
 
-  constructor(model, inputSize, anchorsAnnotated) {
+  constructor(model) {
     this.model = model;
-    this.anchors = anchorsAnnotated.map((anchor) => [anchor.x_center, anchor.y_center]);
+    this.anchors = anchors.anchors.map((anchor) => [anchor.x, anchor.y]);
     this.anchorsTensor = tf.tensor2d(this.anchors);
-    this.inputSize = inputSize;
-    this.inputSizeTensor = tf.tensor1d([inputSize, inputSize]);
-    this.doubleInputSizeTensor = tf.tensor1d([inputSize * 2, inputSize * 2]);
+    this.inputSize = this.model?.inputs[0].shape[2];
+    this.inputSizeTensor = tf.tensor1d([this.inputSize, this.inputSize]);
+    this.doubleInputSizeTensor = tf.tensor1d([this.inputSize * 2, this.inputSize * 2]);
   }
 
   normalizeBoxes(boxes) {

src/handpose/handpipeline.ts

@@ -18,10 +18,10 @@ export class HandPipeline {
   skipped: number;
   detectedHands: number;
 
-  constructor(handDetector, landmarkDetector, inputSize) {
+  constructor(handDetector, landmarkDetector) {
     this.handDetector = handDetector;
     this.landmarkDetector = landmarkDetector;
-    this.inputSize = inputSize;
+    this.inputSize = this.landmarkDetector?.inputs[0].shape[2];
     this.storedBoxes = [];
     this.skipped = 0;
     this.detectedHands = 0;

src/handpose/handpose.ts

@@ -2,9 +2,8 @@ import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as handdetector from './handdetector';
 import * as handpipeline from './handpipeline';
-import * as anchors from './anchors';
 
-const MESH_ANNOTATIONS = {
+const meshAnnotations = {
   thumb: [1, 2, 3, 4],
   indexFinger: [5, 6, 7, 8],
   middleFinger: [9, 10, 11, 12],
@@ -13,49 +12,39 @@ const MESH_ANNOTATIONS = {
   palmBase: [0],
 };
 
-export class HandPose {
-  handPipeline: any;
-
-  constructor(handPipeline) {
-    this.handPipeline = handPipeline;
-  }
-
-  static getAnnotations() {
-    return MESH_ANNOTATIONS;
-  }
-
-  async estimateHands(input, config) {
-    const predictions = await this.handPipeline.estimateHands(input, config);
-    if (!predictions) return [];
-    const hands: Array<{ confidence: number, box: any, boxRaw: any, landmarks: any, annotations: any }> = [];
-    for (const prediction of predictions) {
-      const annotations = {};
-      if (prediction.landmarks) {
-        for (const key of Object.keys(MESH_ANNOTATIONS)) {
-          annotations[key] = MESH_ANNOTATIONS[key].map((index) => prediction.landmarks[index]);
-        }
-      }
-      const box = prediction.box ? [
-        Math.max(0, prediction.box.topLeft[0]),
-        Math.max(0, prediction.box.topLeft[1]),
-        Math.min(input.shape[2], prediction.box.bottomRight[0]) - Math.max(0, prediction.box.topLeft[0]),
-        Math.min(input.shape[1], prediction.box.bottomRight[1]) - Math.max(0, prediction.box.topLeft[1]),
-      ] : [];
-      const boxRaw = [
-        (prediction.box.topLeft[0]) / input.shape[2],
-        (prediction.box.topLeft[1]) / input.shape[1],
-        (prediction.box.bottomRight[0] - prediction.box.topLeft[0]) / input.shape[2],
-        (prediction.box.bottomRight[1] - prediction.box.topLeft[1]) / input.shape[1],
-      ];
-      hands.push({ confidence: Math.round(100 * prediction.confidence) / 100, box, boxRaw, landmarks: prediction.landmarks, annotations });
-    }
-    return hands;
-  }
-}
-
 let handDetectorModel;
 let handPoseModel;
-export async function load(config): Promise<HandPose> {
+let handPipeline;
+
+export async function predict(input, config) {
+  const predictions = await handPipeline.estimateHands(input, config);
+  if (!predictions) return [];
+  const hands: Array<{ confidence: number, box: any, boxRaw: any, landmarks: any, annotations: any }> = [];
+  for (const prediction of predictions) {
+    const annotations = {};
+    if (prediction.landmarks) {
+      for (const key of Object.keys(meshAnnotations)) {
+        annotations[key] = meshAnnotations[key].map((index) => prediction.landmarks[index]);
+      }
+    }
+    const box = prediction.box ? [
+      Math.max(0, prediction.box.topLeft[0]),
+      Math.max(0, prediction.box.topLeft[1]),
+      Math.min(input.shape[2], prediction.box.bottomRight[0]) - Math.max(0, prediction.box.topLeft[0]),
+      Math.min(input.shape[1], prediction.box.bottomRight[1]) - Math.max(0, prediction.box.topLeft[1]),
+    ] : [];
+    const boxRaw = [
+      (prediction.box.topLeft[0]) / input.shape[2],
+      (prediction.box.topLeft[1]) / input.shape[1],
+      (prediction.box.bottomRight[0] - prediction.box.topLeft[0]) / input.shape[2],
+      (prediction.box.bottomRight[1] - prediction.box.topLeft[1]) / input.shape[1],
+    ];
+    hands.push({ confidence: Math.round(100 * prediction.confidence) / 100, box, boxRaw, landmarks: prediction.landmarks, annotations });
+  }
+  return hands;
+}
+
+export async function load(config): Promise<[Object, Object]> {
   if (!handDetectorModel || !handPoseModel) {
     [handDetectorModel, handPoseModel] = await Promise.all([
       config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
@@ -71,8 +60,7 @@ export async function load(config): Promise<HandPose> {
     if (config.debug) log('cached model:', handDetectorModel.modelUrl);
     if (config.debug) log('cached model:', handPoseModel.modelUrl);
   }
-  const handDetector = new handdetector.HandDetector(handDetectorModel, handDetectorModel?.inputs[0].shape[2], anchors.anchors);
-  const handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel, handPoseModel?.inputs[0].shape[2]);
-  const handPose = new HandPose(handPipeline);
-  return handPose;
+  const handDetector = new handdetector.HandDetector(handDetectorModel);
+  handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel);
+  return [handDetectorModel, handPoseModel];
 }

src/human.ts

@@ -82,11 +82,11 @@ export class Human {
   };
   /** Internal: Currently loaded models */
   models: {
-    face: facemesh.MediaPipeFaceMesh | Model | null,
+    face: [Model, Model, Model] | null,
     posenet: Model | null,
     blazepose: Model | null,
     efficientpose: Model | null,
-    handpose: handpose.HandPose | null,
+    handpose: [Model, Model] | null,
     iris: Model | null,
     age: Model | null,
     gender: Model | null,
@@ -431,12 +431,12 @@ export class Human {
       // run handpose
       this.analyze('Start Hand:');
       if (this.config.async) {
-        handRes = this.config.hand.enabled ? this.models.handpose?.estimateHands(process.tensor, this.config) : [];
+        handRes = this.config.hand.enabled ? handpose.predict(process.tensor, this.config) : [];
         if (this.perf.hand) delete this.perf.hand;
       } else {
         this.state = 'run:hand';
         timeStamp = now();
-        handRes = this.config.hand.enabled ? await this.models.handpose?.estimateHands(process.tensor, this.config) : [];
+        handRes = this.config.hand.enabled ? await handpose.predict(process.tensor, this.config) : [];
         current = Math.trunc(now() - timeStamp);
         if (current > 0) this.perf.hand = current;
       }

typedoc/classes/human.html

@@ -445,7 +445,7 @@
 				<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class">
 					<a name="models" class="tsd-anchor"></a>
 					<h3>models</h3>
-					<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> }</span></div>
+					<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> }</span></div>
 					<aside class="tsd-sources">
 					</aside>
 					<div class="tsd-comment tsd-typography">
@@ -472,7 +472,7 @@
 								<h5>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
 							</li>
 							<li class="tsd-parameter">
-								<h5>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span></h5>
+								<h5>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">]</span></h5>
 							</li>
 							<li class="tsd-parameter">
 								<h5>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
@@ -481,7 +481,7 @@
 								<h5>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
 							</li>
 							<li class="tsd-parameter">
-								<h5>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span></h5>
+								<h5>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">]</span></h5>
 							</li>
 							<li class="tsd-parameter">
 								<h5>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>

types/blazeface/blazeface.d.ts

@@ -1,4 +1,3 @@
-export declare const disposeBox: (box: any) => void;
 export declare class BlazeFaceModel {
     model: any;
     anchorsData: any;

types/blazeface/box.d.ts

@@ -15,3 +15,14 @@ export declare function squarifyBox(box: any): {
     endPoint: any[];
     landmarks: any;
 };
+export declare function calculateLandmarksBoundingBox(landmarks: any): {
+    startPoint: number[];
+    endPoint: number[];
+    landmarks: any;
+};
+export declare const disposeBox: (t: any) => void;
+export declare const createBox: (startEndTensor: any) => {
+    startEndTensor: any;
+    startPoint: any;
+    endPoint: any;
+};

types/blazeface/facemesh.d.ts

@@ -1,19 +1,14 @@
-export declare class MediaPipeFaceMesh {
-    facePipeline: any;
-    config: any;
-    constructor(blazeFace: any, blazeMeshModel: any, irisModel: any, config: any);
-    estimateFaces(input: any, config: any): Promise<{
-        confidence: any;
-        boxConfidence: any;
-        faceConfidence: any;
-        box: any;
-        mesh: any;
-        boxRaw: any;
-        meshRaw: any;
-        annotations: any;
-        image: any;
-    }[]>;
-}
-export declare function load(config: any): Promise<MediaPipeFaceMesh>;
+export declare function predict(input: any, config: any): Promise<{
+    confidence: any;
+    boxConfidence: any;
+    faceConfidence: any;
+    box: any;
+    mesh: any;
+    boxRaw: any;
+    meshRaw: any;
+    annotations: any;
+    image: any;
+}[]>;
+export declare function load(config: any): Promise<[Object, Object, Object]>;
 export declare const triangulation: number[];
 export declare const uvmap: number[][];

types/blazeface/facepipeline.d.ts

@@ -27,9 +27,4 @@ export declare class Pipeline {
     };
     getAdjustedIrisCoords(rawCoords: any, irisCoords: any, direction: any): any;
     predict(input: any, config: any): Promise<any>;
-    calculateLandmarksBoundingBox(landmarks: any): {
-        startPoint: number[];
-        endPoint: number[];
-        landmarks: any;
-    };
 }

types/blazeface/util.d.ts

@@ -19,3 +19,4 @@ export declare function buildRotationMatrix(rotation: any, center: any): number[
 export declare function invertTransformMatrix(matrix: any): any[][];
 export declare function rotatePoint(homogeneousCoordinate: any, rotationMatrix: any): number[];
 export declare function xyDistanceBetweenPoints(a: any, b: any): number;
+export declare function generateAnchors(inputSize: any): [number, number][];

types/handpose/anchors.d.ts

@@ -1,6 +1,4 @@
 export declare const anchors: {
-    w: number;
-    h: number;
-    x_center: number;
-    y_center: number;
+    x: number;
+    y: number;
 }[];

types/handpose/handdetector.d.ts

@@ -5,7 +5,7 @@ export declare class HandDetector {
     inputSize: number;
     inputSizeTensor: any;
     doubleInputSizeTensor: any;
-    constructor(model: any, inputSize: any, anchorsAnnotated: any);
+    constructor(model: any);
     normalizeBoxes(boxes: any): any;
     normalizeLandmarks(rawPalmLandmarks: any, index: any): any;
     getBoxes(input: any, config: any): Promise<{

types/handpose/handpipeline.d.ts

@@ -5,7 +5,7 @@ export declare class HandPipeline {
     storedBoxes: any;
     skipped: number;
     detectedHands: number;
-    constructor(handDetector: any, landmarkDetector: any, inputSize: any);
+    constructor(handDetector: any, landmarkDetector: any);
     getBoxForPalmLandmarks(palmLandmarks: any, rotationMatrix: any): {
         startPoint: number[];
         endPoint: any[];

types/handpose/handpose.d.ts

@@ -1,20 +1,8 @@
-export declare class HandPose {
-    handPipeline: any;
-    constructor(handPipeline: any);
-    static getAnnotations(): {
-        thumb: number[];
-        indexFinger: number[];
-        middleFinger: number[];
-        ringFinger: number[];
-        pinky: number[];
-        palmBase: number[];
-    };
-    estimateHands(input: any, config: any): Promise<{
-        confidence: number;
-        box: any;
-        boxRaw: any;
-        landmarks: any;
-        annotations: any;
-    }[]>;
-}
-export declare function load(config: any): Promise<HandPose>;
+export declare function predict(input: any, config: any): Promise<{
+    confidence: number;
+    box: any;
+    boxRaw: any;
+    landmarks: any;
+    annotations: any;
+}[]>;
+export declare function load(config: any): Promise<[Object, Object]>;

types/human.d.ts

@@ -73,11 +73,11 @@ export declare class Human {
     };
     /** Internal: Currently loaded models */
     models: {
-        face: facemesh.MediaPipeFaceMesh | Model | null;
+        face: [Model, Model, Model] | null;
         posenet: Model | null;
         blazepose: Model | null;
         efficientpose: Model | null;
-        handpose: handpose.HandPose | null;
+        handpose: [Model, Model] | null;
         iris: Model | null;
         age: Model | null;
         gender: Model | null;