start working on efficientpose

CHANGELOG.md

@@ -9,6 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
+### **HEAD -> main** 2021/03/26 mandic00@live.com
+
+
 ### **1.2.5** 2021/03/25 mandic00@live.com
 
 - fix broken exports

demo/browser.js

@@ -3,22 +3,20 @@ import Human from '../src/human';
 import Menu from './menu.js';
 import GLBench from './gl-bench.js';
 
-const userConfig = { backend: 'webgl' }; // add any user configuration overrides
+// const userConfig = { backend: 'webgl' }; // add any user configuration overrides
 
-/*
 const userConfig = {
-  backend: 'wasm',
+  backend: 'webgl',
   async: false,
   warmup: 'full',
   videoOptimized: true,
   filter: { enabled: true },
-  face: { enabled: true, mesh: { enabled: true }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
+  face: { enabled: false, mesh: { enabled: false }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
   hand: { enabled: false },
   gesture: { enabled: false },
-  body: { enabled: false, modelPath: '../models/blazepose.json' },
+  body: { enabled: true, modelPath: '../models/efficientpose.json' },
   object: { enabled: false },
 };
-*/
 
 const human = new Human(userConfig);
 

package.json

@@ -56,14 +56,14 @@
     "@tensorflow/tfjs-layers": "^3.3.0",
     "@tensorflow/tfjs-node": "^3.3.0",
     "@tensorflow/tfjs-node-gpu": "^3.3.0",
-    "@types/node": "^14.14.35",
+    "@types/node": "^14.14.36",
     "@typescript-eslint/eslint-plugin": "^4.19.0",
     "@typescript-eslint/parser": "^4.19.0",
     "@vladmandic/pilogger": "^0.2.15",
     "chokidar": "^3.5.1",
     "dayjs": "^1.10.4",
-    "esbuild": "^0.10.0",
+    "esbuild": "^0.10.1",
-    "eslint": "^7.22.0",
+    "eslint": "^7.23.0",
     "eslint-config-airbnb-base": "^14.2.1",
     "eslint-plugin-import": "^2.22.1",
     "eslint-plugin-json": "^2.1.2",

src/blazeface/facemesh.ts

@@ -59,7 +59,7 @@ export class MediaPipeFaceMesh {
 }
 
 let faceModels = [null, null, null];
-export async function load(config) {
+export async function load(config): Promise<MediaPipeFaceMesh> {
   // @ts-ignore
   faceModels = await Promise.all([
     (!faceModels[0] && config.face.enabled) ? blazeface.load(config) : null,

src/draw/draw.ts

@@ -221,12 +221,14 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
     }
     if (drawOptions.drawLabels) {
       ctx.font = drawOptions.font;
+      if (result[i].keypoints) {
         for (const pt of result[i].keypoints) {
           ctx.fillStyle = drawOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : drawOptions.color;
           ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4);
         }
       }
-    if (drawOptions.drawPolygons) {
+    }
+    if (drawOptions.drawPolygons && result[i].keypoints) {
       let part;
       const points: any[] = [];
       // torso

src/efficientpose/efficientpose.ts

@@ -0,0 +1,97 @@
+import { log } from '../helpers';
+import * as tf from '../../dist/tfjs.esm.js';
+import * as profile from '../profile';
+
+let model;
+let last = { };
+let skipped = Number.MAX_SAFE_INTEGER;
+
+const bodyParts = ['head', 'neck', 'rightShoulder', 'rightElbow', 'rightWrist', 'chest', 'leftShoulder', 'leftElbow', 'leftWrist', 'pelvis', 'rightHip', 'rightKnee', 'rightAnkle', 'leftHip', 'leftKnee', 'leftAnkle'];
+
+export async function load(config) {
+  if (!model) {
+    model = await tf.loadGraphModel(config.body.modelPath);
+    if (config.debug) log(`load model: ${config.body.modelPath.match(/\/(.*)\./)[1]}`);
+  }
+  return model;
+}
+
+// performs argmax and max functions on a 2d tensor
+function max2d(inputs, minScore) {
+  const [width, height] = inputs.shape;
+  return tf.tidy(() => {
+    // modulus op implemented in tf
+    const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32')));
+    // combine all data
+    const reshaped = tf.reshape(inputs, [height * width]);
+    // get highest score
+    const score = tf.max(reshaped, 0).dataSync()[0];
+    if (score > minScore) {
+      // skip coordinate calculation is score is too low
+      const coords = tf.argMax(reshaped, 0);
+      const x = mod(coords, width).dataSync()[0];
+      const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0];
+      return [x, y, score];
+    }
+    return [0, 0, score];
+  });
+}
+
+export async function predict(image, config) {
+  if (!model) return null;
+  if ((skipped < config.body.skipFrames) && config.videoOptimized && Object.keys(last).length > 0) {
+    skipped++;
+    return last;
+  }
+  if (config.videoOptimized) skipped = 0;
+  else skipped = Number.MAX_SAFE_INTEGER;
+  return new Promise(async (resolve) => {
+    const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
+    const enhance = tf.mul(resize, [255.0]);
+    tf.dispose(resize);
+
+    let resT;
+
+    if (!config.profile) {
+      if (config.body.enabled) resT = await model.predict(enhance);
+    } else {
+      const profileT = config.body.enabled ? await tf.profile(() => model.predict(enhance)) : {};
+      resT = profileT.result.clone();
+      profileT.result.dispose();
+      profile.run('body', profileT);
+    }
+    enhance.dispose();
+
+    if (resT) {
+      const parts: Array<{ id, score, part, position: { x, y }, positionRaw: { xRaw, yRaw} }> = [];
+      const squeeze = resT.squeeze();
+      tf.dispose(resT);
+      // body parts are basically just a stack of 2d tensors
+      const stack = squeeze.unstack(2);
+      tf.dispose(squeeze);
+      // process each unstacked tensor as a separate body part
+      for (let id = 0; id < stack.length; id++) {
+        // actual processing to get coordinates and score
+        const [x, y, score] = max2d(stack[id], config.body.scoreThreshold);
+        if (score > config.body.scoreThreshold) {
+          parts.push({
+            id,
+            score,
+            part: bodyParts[id],
+            positionRaw: {
+              xRaw: x / model.inputs[0].shape[2], // x normalized to 0..1
+              yRaw: y / model.inputs[0].shape[1], // y normalized to 0..1
+            },
+            position: {
+              x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), // x normalized to input image size
+              y: Math.round(image.shape[1] * y / model.inputs[0].shape[1]), // y normalized to input image size
+            },
+          });
+        }
+      }
+      stack.forEach((s) => tf.dispose(s));
+      last = parts;
+    }
+    resolve(last);
+  });
+}

src/handpose/handpose.ts

@@ -55,7 +55,7 @@ export class HandPose {
   }
 }
 
-export async function load(config) {
+export async function load(config): Promise<HandPose> {
   const [handDetectorModel, handPoseModel] = await Promise.all([
     config.hand.enabled ? tf.loadGraphModel(config.hand.detector.modelPath, { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
     config.hand.landmarks ? tf.loadGraphModel(config.hand.skeleton.modelPath, { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,

src/human.ts

@@ -12,6 +12,7 @@ import * as embedding from './embedding/embedding';
 import * as posenet from './posenet/posenet';
 import * as handpose from './handpose/handpose';
 import * as blazepose from './blazepose/blazepose';
+import * as efficientpose from './efficientpose/efficientpose';
 import * as nanodet from './nanodet/nanodet';
 import * as gesture from './gesture/gesture';
 import * as image from './image/image';
@@ -62,9 +63,10 @@ export class Human {
   };
   // models
   models: {
-    face: facemesh.MediaPipeFaceMesh | null,
+    face: facemesh.MediaPipeFaceMesh | Model | null,
     posenet: posenet.PoseNet | null,
     blazepose: Model | null,
+    efficientpose: Model | null,
     handpose: handpose.HandPose | null,
     iris: Model | null,
     age: Model | null,
@@ -108,6 +110,7 @@ export class Human {
       face: null,
       posenet: null,
       blazepose: null,
+      efficientpose: null,
       handpose: null,
       iris: null,
       age: null,
@@ -206,9 +209,12 @@ export class Human {
         this.models.gender,
         this.models.emotion,
         this.models.embedding,
+        // @ts-ignore
         this.models.handpose,
+        // @ts-ignore false warning with latest @typescript-eslint
         this.models.posenet,
         this.models.blazepose,
+        this.models.efficientpose,
         this.models.nanodet,
         this.models.faceres,
       ] = await Promise.all([
@@ -217,9 +223,10 @@ export class Human {
         this.models.gender || ((this.config.face.enabled && this.config.face.gender.enabled) ? gender.load(this.config) : null),
         this.models.emotion || ((this.config.face.enabled && this.config.face.emotion.enabled) ? emotion.load(this.config) : null),
         this.models.embedding || ((this.config.face.enabled && this.config.face.embedding.enabled) ? embedding.load(this.config) : null),
-        this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null),
+        this.models.handpose || (this.config.hand.enabled ? <Promise<handpose.HandPose>>handpose.load(this.config) : null),
         this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null),
-        this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
+        this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
+        this.models.efficientpose || (this.config.body.enabled && this.config.body.modelPath.includes('efficientpose') ? efficientpose.load(this.config) : null),
         this.models.nanodet || (this.config.object.enabled ? nanodet.load(this.config) : null),
         this.models.faceres || ((this.config.face.enabled && this.config.face.description.enabled) ? faceres.load(this.config) : null),
       ]);
@@ -232,6 +239,7 @@ export class Human {
       if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config);
       if (this.config.body.enabled && !this.models.posenet && this.config.body.modelPath.includes('posenet')) this.models.posenet = await posenet.load(this.config);
       if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes('blazepose')) this.models.blazepose = await blazepose.load(this.config);
+      if (this.config.body.enabled && !this.models.efficientpose && this.config.body.modelPath.includes('efficientpose')) this.models.efficientpose = await efficientpose.load(this.config);
       if (this.config.object.enabled && !this.models.nanodet) this.models.nanodet = await nanodet.load(this.config);
       if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) this.models.faceres = await faceres.load(this.config);
     }
@@ -363,13 +371,15 @@ export class Human {
       this.analyze('Start Body:');
       if (this.config.async) {
         if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
-        else bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : [];
         if (this.perf.body) delete this.perf.body;
       } else {
         this.state = 'run:body';
         timeStamp = now();
         if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
-        else bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : [];
         current = Math.trunc(now() - timeStamp);
         if (current > 0) this.perf.body = current;
       }

src/prototype/prototype.ts

@@ -0,0 +1,61 @@
+import { log } from '../helpers';
+import * as tf from '../../dist/tfjs.esm.js';
+import * as profile from '../profile';
+
+/*
+Prototype implementation for model processing
+Must implement
+- load()
+- predict()
+Must account for:
+- image processing, tfjs profiling
+*/
+
+let model;
+let last = { };
+let skipped = Number.MAX_SAFE_INTEGER;
+
+export async function load(config) {
+  if (!model) {
+    model = await tf.loadGraphModel(config.prototype.modelPath);
+    if (config.debug) log(`load model: ${config.prototype.modelPath.match(/\/(.*)\./)[1]}`);
+  }
+  return model;
+}
+
+export async function predict(image, config) {
+  if (!model) return null;
+  if ((skipped < config.prototype.skipFrames) && config.videoOptimized && Object.keys(last).length > 0) {
+    skipped++;
+    return last;
+  }
+  if (config.videoOptimized) skipped = 0;
+  else skipped = Number.MAX_SAFE_INTEGER;
+  return new Promise(async (resolve) => {
+    const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
+    const enhance = tf.mul(resize, [255.0]);
+    tf.dispose(resize);
+
+    let resT;
+
+    if (!config.profile) {
+      if (config.prototype.enabled) resT = await model.predict(enhance);
+    } else {
+      const profileT = config.prototype.enabled ? await tf.profile(() => model.predict(enhance)) : {};
+      resT = profileT.result.clone();
+      profileT.result.dispose();
+      profile.run('prototype', profileT);
+    }
+    enhance.dispose();
+
+    let obj = {};
+    if (resT) {
+      const data = resT.dataSync();
+      obj = { data };
+      tf.dispose(resT);
+    }
+
+    last = obj;
+    resolve(obj);
+  });
+}

typedoc/classes/human.html

@@ -425,7 +425,7 @@
 				<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class">
 					<a name="models" class="tsd-anchor"></a>
 					<h3>models</h3>
-					<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">PoseNet</span><span class="tsd-signature-symbol"> }</span></div>
+					<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">PoseNet</span><span class="tsd-signature-symbol"> }</span></div>
 					<aside class="tsd-sources">
 					</aside>
 					<div class="tsd-type-declaration">
@@ -437,6 +437,9 @@
 							<li class="tsd-parameter">
 								<h5>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
 							</li>
+							<li class="tsd-parameter">
+								<h5>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
+							</li>
 							<li class="tsd-parameter">
 								<h5>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
 							</li>
@@ -444,7 +447,7 @@
 								<h5>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
 							</li>
 							<li class="tsd-parameter">
-								<h5>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span></h5>
+								<h5>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span></h5>
 							</li>
 							<li class="tsd-parameter">
 								<h5>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>

types/efficientpose/efficientpose.d.ts

@@ -0,0 +1,2 @@
+export declare function load(config: any): Promise<any>;
+export declare function predict(image: any, config: any): Promise<unknown>;

types/human.d.ts

@@ -54,9 +54,10 @@ export declare class Human {
         all: typeof draw.all;
     };
     models: {
-        face: facemesh.MediaPipeFaceMesh | null;
+        face: facemesh.MediaPipeFaceMesh | Model | null;
         posenet: posenet.PoseNet | null;
         blazepose: Model | null;
+        efficientpose: Model | null;
         handpose: handpose.HandPose | null;
         iris: Model | null;
         age: Model | null;