From 7f82eb58c55fc9855e575813020b2de564efd3e9 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Fri, 31 Dec 2021 13:58:03 -0500
Subject: [PATCH] update blazepose

---
 src/body/blazepose.ts   |  4 +++-
 src/result.ts           |  2 ++
 src/util/interpolate.ts | 11 ++++++++---
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/body/blazepose.ts b/src/body/blazepose.ts
index 7b30c010..f60726e9 100644
--- a/src/body/blazepose.ts
+++ b/src/body/blazepose.ts
@@ -147,6 +147,7 @@ async function detectLandmarks(input: Tensor, config: Config, outputSize: [numbe
   [t.ld/* 1,195(39*5) */, t.segmentation/* 1,256,256,1 */, t.heatmap/* 1,64,64,39 */, t.world/* 1,117(39*3) */, t.poseflag/* 1,1 */] = models.landmarks?.execute(input, outputNodes.landmarks) as Tensor[]; // run model
   const poseScore = (await t.poseflag.data())[0];
   const points = await t.ld.data();
+  const distances = await t.world.data();
   Object.keys(t).forEach((tensor) => tf.dispose(t[tensor])); // dont need tensors after this
   const keypointsRelative: Array<BodyKeypoint> = [];
   const depth = 5; // each points has x,y,z,visibility,presence
@@ -156,7 +157,8 @@ async function detectLandmarks(input: Tensor, config: Config, outputSize: [numbe
     const adjScore = Math.trunc(100 * score * presence * poseScore) / 100;
     const positionRaw: Point = [points[depth * i + 0] / inputSize.landmarks[0], points[depth * i + 1] / inputSize.landmarks[1], points[depth * i + 2] + 0];
     const position: Point = [Math.trunc(outputSize[0] * positionRaw[0]), Math.trunc(outputSize[1] * positionRaw[1]), positionRaw[2] as number];
-    keypointsRelative.push({ part: coords.kpt[i] as BodyLandmark, positionRaw, position, score: adjScore });
+    const distance: Point = [distances[depth * i + 0], distances[depth * i + 1], distances[depth * i + 2] + 0];
+    keypointsRelative.push({ part: coords.kpt[i] as BodyLandmark, positionRaw, position, distance, score: adjScore });
   }
   if (poseScore < (config.body.minConfidence || 0)) return null;
   fixKeypoints(keypointsRelative);
diff --git a/src/result.ts b/src/result.ts
index 71682cca..e7f749fd 100644
--- a/src/result.ts
+++ b/src/result.ts
@@ -90,6 +90,8 @@ export interface BodyKeypoint {
   position: Point,
   /** body part position normalized to 0..1 */
   positionRaw: Point,
+  /** body part position relative to body center in meters */
+  distance?: Point,
   /** body part detection score */
   score: number,
 }
diff --git a/src/util/interpolate.ts b/src/util/interpolate.ts
index aa64ecea..a8728723 100644
--- a/src/util/interpolate.ts
+++ b/src/util/interpolate.ts
@@ -53,9 +53,14 @@ export function calc(newResult: Result, config: Config): Result {
             bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[2] || 0) + (newKpt.position[2] || 0)) / bufferedFactor : newKpt.position[2],
           ],
           positionRaw: [
-            bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[0] || 0) + (newKpt.positionRaw[0] || 0)) / bufferedFactor : newKpt.position[0],
-            bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[1] || 0) + (newKpt.positionRaw[1] || 0)) / bufferedFactor : newKpt.position[1],
-            bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[2] || 0) + (newKpt.positionRaw[2] || 0)) / bufferedFactor : newKpt.position[2],
+            bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[0] || 0) + (newKpt.positionRaw[0] || 0)) / bufferedFactor : newKpt.positionRaw[0],
+            bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[1] || 0) + (newKpt.positionRaw[1] || 0)) / bufferedFactor : newKpt.positionRaw[1],
+            bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[2] || 0) + (newKpt.positionRaw[2] || 0)) / bufferedFactor : newKpt.positionRaw[2],
+          ],
+          distance: [
+            bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].distance?.[0] || 0) + (newKpt.distance?.[0] || 0)) / bufferedFactor : newKpt.distance?.[0],
+            bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].distance?.[1] || 0) + (newKpt.distance?.[1] || 0)) / bufferedFactor : newKpt.distance?.[1],
+            bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].distance?.[2] || 0) + (newKpt.distance?.[2] || 0)) / bufferedFactor : newKpt.distance?.[2],
           ],
         }))) as Array<{ score: number, part: BodyLandmark, position: [number, number, number?], positionRaw: [number, number, number?] }>;