From 32247c1b0649a7a41b0a0dc27c789c9d343bc1c9 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Fri, 5 Mar 2021 08:03:00 -0500
Subject: [PATCH] cleanup blazepose code

---
 src/blazepose/blazepose.ts | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/blazepose/blazepose.ts b/src/blazepose/blazepose.ts
index 80996a79..f5d97efb 100644
--- a/src/blazepose/blazepose.ts
+++ b/src/blazepose/blazepose.ts
@@ -8,6 +8,7 @@ let model;
 export async function load(config) {
   if (!model) {
     model = await tf.loadGraphModel(config.body.modelPath);
+    // blazepose inputSize is 256x256px, but we can find that out dynamically
     model.width = parseInt(model.signature.inputs['input_1:0'].tensorShape.dim[2].size);
     model.height = parseInt(model.signature.inputs['input_1:0'].tensorShape.dim[1].size);
     if (config.debug) log(`load model: ${config.body.modelPath.match(/\/(.*)\./)[1]}`);
@@ -25,31 +26,29 @@ export async function predict(image, config) {
   // let segmentation; // not used right now since we have keypoints and don't need to go through matrix using strides
   // let poseflag; // irrelevant
   let points;
-  if (!config.profile) {
+  if (!config.profile) { // run through profiler or just execute
     const resT = await model.predict(normalize);
     // segmentation = resT[0].dataSync();
     // poseflag = resT[1].dataSync();
-    points = resT.find((t) => (t.size === 195 || t.size === 155)).dataSync();
+    points = resT.find((t) => (t.size === 195 || t.size === 155)).dataSync(); // order of output tensors may change between models, full has 195 and upper has 155 items
     resT.forEach((t) => t.dispose());
   } else {
     const profileData = await tf.profile(() => model.predict(normalize));
-    // segmentation = profileData.result[0].dataSync();
-    // poseflag = profileData.result[1].dataSync();
-    points = profileData.result.find((t) => t.size === 195).dataSync(); // find a tensor with 195 items which is 39 points with 5 properties
+    points = profileData.result.find((t) => (t.size === 195 || t.size === 155)).dataSync();
     profileData.result.forEach((t) => t.dispose());
     profile.run('blazepose', profileData);
   }
   normalize.dispose();
   const keypoints: Array<{ id, part, position: { x, y, z }, score, presence }> = [];
-  const labels = points.length === 195 ? annotations.full : annotations.upper;
-  const depth = 5;
+  const labels = points.length === 195 ? annotations.full : annotations.upper; // full model has 39 keypoints, upper has 31 keypoints
+  const depth = 5; // each points has x,y,z,visibility,presence
   for (let i = 0; i < points.length / depth; i++) {
     keypoints.push({
       id: i,
       part: labels[i],
       position: {
-        x: Math.trunc(imgSize.width * points[depth * i + 0] / 255),
-        y: Math.trunc(imgSize.height * points[depth * i + 1] / 255),
+        x: Math.trunc(imgSize.width * points[depth * i + 0] / 255), // return normalized x value istead of 0..255
+        y: Math.trunc(imgSize.height * points[depth * i + 1] / 255), // return normalized y value istead of 0..255
         z: Math.trunc(points[depth * i + 2]) + 0, // fix negative zero
       },
       score: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 3])))) / 100, // reverse sigmoid value
@@ -61,6 +60,9 @@ export async function predict(image, config) {
 }
 
 /*
-Model card: https://drive.google.com/file/d/10IU-DRP2ioSNjKFdiGbmmQX81xAYj88s/view
-Download: https://github.com/PINTO0309/PINTO_model_zoo/tree/main/058_BlazePose_Full_Keypoints
+Model card:
+- https://drive.google.com/file/d/10IU-DRP2ioSNjKFdiGbmmQX81xAYj88s/view
+Download:
+- https://github.com/PINTO0309/PINTO_model_zoo/tree/main/058_BlazePose_Full_Keypoints/10_new_256x256/saved_model/tfjs_model_float16
+- https://github.com/PINTO0309/PINTO_model_zoo/tree/main/053_BlazePose/20_new_256x256/saved_model/tfjs_model_float16
 */