From 5a6ef389a62d13689ab4ad98017b5edec8d76b47 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 29 Dec 2021 12:37:46 -0500 Subject: [PATCH] update blazepose --- src/body/blazepose.ts | 13 +++++++++++++ src/body/blazeposecoords.ts | 33 ++++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/body/blazepose.ts b/src/body/blazepose.ts index 8208a186..7b30c010 100644 --- a/src/body/blazepose.ts +++ b/src/body/blazepose.ts @@ -123,6 +123,18 @@ function rescaleKeypoints(keypoints: Array, outputSize: [number, n return keypoints; } +async function fixKeypoints(keypoints: Array) { + // palm z-coord is incorrect around near-zero so we approximate it + const leftPalm = keypoints.find((k) => k.part === 'leftPalm') as BodyKeypoint; + const leftWrist = keypoints.find((k) => k.part === 'leftWrist') as BodyKeypoint; + const leftIndex = keypoints.find((k) => k.part === 'leftIndex') as BodyKeypoint; + leftPalm.position[2] = ((leftWrist.position[2] || 0) + (leftIndex.position[2] || 0)) / 2; + const rightPalm = keypoints.find((k) => k.part === 'rightPalm') as BodyKeypoint; + const rightWrist = keypoints.find((k) => k.part === 'rightWrist') as BodyKeypoint; + const rightIndex = keypoints.find((k) => k.part === 'rightIndex') as BodyKeypoint; + rightPalm.position[2] = ((rightWrist.position[2] || 0) + (rightIndex.position[2] || 0)) / 2; +} + async function detectLandmarks(input: Tensor, config: Config, outputSize: [number, number]): Promise { /** * t.ld: 39 keypoints [x,y,z,score,presence] normalized to input size @@ -147,6 +159,7 @@ async function detectLandmarks(input: Tensor, config: Config, outputSize: [numbe keypointsRelative.push({ part: coords.kpt[i] as BodyLandmark, positionRaw, position, score: adjScore }); } if (poseScore < (config.body.minConfidence || 0)) return null; + fixKeypoints(keypointsRelative); const keypoints: Array = rescaleKeypoints(keypointsRelative, outputSize); // keypoints were relative to input image which is padded const kpts = keypoints.map((k) => k.position); const boxes = box.calc(kpts, [outputSize[0], outputSize[1]]); // now find boxes based on rescaled keypoints diff --git a/src/body/blazeposecoords.ts b/src/body/blazeposecoords.ts index 75a9a37d..988cd1b1 100644 --- a/src/body/blazeposecoords.ts +++ b/src/body/blazeposecoords.ts @@ -43,14 +43,29 @@ export const kpt: Array = [ ]; export const connected: Record = { - leftLeg: ['leftHip', 'leftKnee', 'leftAnkle', 'leftHeel', 'leftFoot'], - rightLeg: ['rightHip', 'rightKnee', 'rightAnkle', 'rightHeel', 'rightFoot'], - torso: ['leftShoulder', 'rightShoulder', 'rightHip', 'leftHip', 'leftShoulder', 'rightShoulder'], - leftArm: ['leftShoulder', 'leftElbow', 'leftWrist', 'leftPalm'], - rightArm: ['rightShoulder', 'rightElbow', 'rightWrist', 'rightPalm'], - leftEye: ['leftEyeInside', 'leftEye', 'leftEyeOutside'], - rightEye: ['rightEyeInside', 'rightEye', 'rightEyeOutside'], + shoulders: ['leftShoulder', 'rightShoulder'], + hips: ['rightHip', 'leftHip'], mouth: ['leftMouth', 'rightMouth'], - // leftHand: ['leftHand', 'leftPalm', 'leftPinky', 'leftPalm', 'leftIndex', 'leftPalm', 'leftThumb'], - // rightHand: ['rightHand', 'rightPalm', 'rightPinky', 'rightPalm', 'rightIndex', 'rightPalm', 'rightThumb'], + leftLegUpper: ['leftHip', 'leftKnee'], + leftLegLower: ['leftKnee', 'leftAnkle'], + leftFoot: ['leftAnkle', 'leftHeel', 'leftFoot'], + leftTorso: ['leftShoulder', 'leftHip'], + leftArmUpper: ['leftShoulder', 'leftElbow'], + leftArmLower: ['leftElbow', 'leftWrist'], + leftHand: ['leftWrist', 'leftPalm'], + leftHandPinky: ['leftPalm', 'leftPinky'], + leftHandIndex: ['leftPalm', 'leftIndex'], + leftHandThumb: ['leftPalm', 'leftThumb'], + leftEyeOutline: ['leftEyeInside', 'leftEyeOutside'], + rightLegUpper: ['rightHip', 'rightKnee'], + rightLegLower: ['rightKnee', 'rightAnkle'], + rightFoot: ['rightAnkle', 'rightHeel', 'rightFoot'], + rightTorso: ['rightShoulder', 'rightHip'], + rightArmUpper: ['rightShoulder', 'rightElbow'], + rightArmLower: ['rightElbow', 'rightWrist'], + rightHand: ['rightWrist', 'rightPalm'], + rightHandPinky: ['rightPalm', 'rightPinky'], + rightHandIndex: ['rightPalm', 'rightIndex'], + rightHandThumb: ['rightPalm', 'rightThumb'], + rightEyeOutline: ['rightEyeInside', 'rightEyeOutside'], };