update blazepose

src/body/blazepose.ts

@@ -123,6 +123,18 @@ function rescaleKeypoints(keypoints: Array<BodyKeypoint>, outputSize: [number, n
   return keypoints;
 }
 
+async function fixKeypoints(keypoints: Array<BodyKeypoint>) {
+  // palm z-coord is incorrect around near-zero so we approximate it
+  const leftPalm = keypoints.find((k) => k.part === 'leftPalm') as BodyKeypoint;
+  const leftWrist = keypoints.find((k) => k.part === 'leftWrist') as BodyKeypoint;
+  const leftIndex = keypoints.find((k) => k.part === 'leftIndex') as BodyKeypoint;
+  leftPalm.position[2] = ((leftWrist.position[2] || 0) + (leftIndex.position[2] || 0)) / 2;
+  const rightPalm = keypoints.find((k) => k.part === 'rightPalm') as BodyKeypoint;
+  const rightWrist = keypoints.find((k) => k.part === 'rightWrist') as BodyKeypoint;
+  const rightIndex = keypoints.find((k) => k.part === 'rightIndex') as BodyKeypoint;
+  rightPalm.position[2] = ((rightWrist.position[2] || 0) + (rightIndex.position[2] || 0)) / 2;
+}
+
 async function detectLandmarks(input: Tensor, config: Config, outputSize: [number, number]): Promise<BodyResult | null> {
   /**
    * t.ld: 39 keypoints [x,y,z,score,presence] normalized to input size
@@ -147,6 +159,7 @@ async function detectLandmarks(input: Tensor, config: Config, outputSize: [numbe
     keypointsRelative.push({ part: coords.kpt[i] as BodyLandmark, positionRaw, position, score: adjScore });
   }
   if (poseScore < (config.body.minConfidence || 0)) return null;
+  fixKeypoints(keypointsRelative);
   const keypoints: Array<BodyKeypoint> = rescaleKeypoints(keypointsRelative, outputSize); // keypoints were relative to input image which is padded
   const kpts = keypoints.map((k) => k.position);
   const boxes = box.calc(kpts, [outputSize[0], outputSize[1]]); // now find boxes based on rescaled keypoints

src/body/blazeposecoords.ts

@@ -43,14 +43,29 @@ export const kpt: Array<string> = [
 ];
 
 export const connected: Record<string, string[]> = {
-  leftLeg: ['leftHip', 'leftKnee', 'leftAnkle', 'leftHeel', 'leftFoot'],
-  rightLeg: ['rightHip', 'rightKnee', 'rightAnkle', 'rightHeel', 'rightFoot'],
-  torso: ['leftShoulder', 'rightShoulder', 'rightHip', 'leftHip', 'leftShoulder', 'rightShoulder'],
-  leftArm: ['leftShoulder', 'leftElbow', 'leftWrist', 'leftPalm'],
-  rightArm: ['rightShoulder', 'rightElbow', 'rightWrist', 'rightPalm'],
-  leftEye: ['leftEyeInside', 'leftEye', 'leftEyeOutside'],
-  rightEye: ['rightEyeInside', 'rightEye', 'rightEyeOutside'],
+  shoulders: ['leftShoulder', 'rightShoulder'],
+  hips: ['rightHip', 'leftHip'],
   mouth: ['leftMouth', 'rightMouth'],
-  // leftHand: ['leftHand', 'leftPalm', 'leftPinky', 'leftPalm', 'leftIndex', 'leftPalm', 'leftThumb'],
-  // rightHand: ['rightHand', 'rightPalm', 'rightPinky', 'rightPalm', 'rightIndex', 'rightPalm', 'rightThumb'],
+  leftLegUpper: ['leftHip', 'leftKnee'],
+  leftLegLower: ['leftKnee', 'leftAnkle'],
+  leftFoot: ['leftAnkle', 'leftHeel', 'leftFoot'],
+  leftTorso: ['leftShoulder', 'leftHip'],
+  leftArmUpper: ['leftShoulder', 'leftElbow'],
+  leftArmLower: ['leftElbow', 'leftWrist'],
+  leftHand: ['leftWrist', 'leftPalm'],
+  leftHandPinky: ['leftPalm', 'leftPinky'],
+  leftHandIndex: ['leftPalm', 'leftIndex'],
+  leftHandThumb: ['leftPalm', 'leftThumb'],
+  leftEyeOutline: ['leftEyeInside', 'leftEyeOutside'],
+  rightLegUpper: ['rightHip', 'rightKnee'],
+  rightLegLower: ['rightKnee', 'rightAnkle'],
+  rightFoot: ['rightAnkle', 'rightHeel', 'rightFoot'],
+  rightTorso: ['rightShoulder', 'rightHip'],
+  rightArmUpper: ['rightShoulder', 'rightElbow'],
+  rightArmLower: ['rightElbow', 'rightWrist'],
+  rightHand: ['rightWrist', 'rightPalm'],
+  rightHandPinky: ['rightPalm', 'rightPinky'],
+  rightHandIndex: ['rightPalm', 'rightIndex'],
+  rightHandThumb: ['rightPalm', 'rightThumb'],
+  rightEyeOutline: ['rightEyeInside', 'rightEyeOutside'],
 };