diff --git a/src/draw/draw.ts b/src/draw/draw.ts index 4ba33295..eb386837 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -293,16 +293,16 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, dra } if (localOptions.drawPoints) { for (let pt = 0; pt < result[i].keypoints.length; pt++) { - ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position.z || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position.z || 0))}, 255, 0.5)` : localOptions.color; - point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions); + ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position[2] ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position[2] || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position[2] || 0))}, 255, 0.5)` : localOptions.color; + point(ctx, result[i].keypoints[pt].position[0], result[i].keypoints[pt].position[1], 0, localOptions); } } if (localOptions.drawLabels) { ctx.font = localOptions.font; if (result[i].keypoints) { for (const pt of result[i].keypoints) { - ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : localOptions.color; - ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position.x + 4, pt.position.y + 4); + ctx.fillStyle = localOptions.useDepth && pt.position[2] ? `rgba(${127.5 + (2 * pt.position[2])}, ${127.5 - (2 * pt.position[2])}, 255, 0.5)` : localOptions.color; + ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position[0] + 4, pt.position[1] + 4); } } } @@ -312,68 +312,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, dra // shoulder line points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); curves(ctx, points, localOptions); // torso main points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'rightHip'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'leftHip'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso // leg left points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftHip'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'leftKnee'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'leftHeel'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'leftFoot'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); curves(ctx, points, localOptions); // leg right points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightHip'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'rightKnee'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'rightHeel'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'rightFoot'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); curves(ctx, points, localOptions); // arm left points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'leftElbow'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'leftWrist'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'leftPalm'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); curves(ctx, points, localOptions); // arm right points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'rightElbow'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'rightWrist'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); part = result[i].keypoints.find((a) => a.part === 'rightPalm'); - if (part) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position[0], part.position[1]]); curves(ctx, points, localOptions); // draw all } diff --git a/src/interpolate.ts b/src/interpolate.ts index d657c1f8..4739f80b 100644 --- a/src/interpolate.ts +++ b/src/interpolate.ts @@ -22,15 +22,19 @@ export function calc(newResult: Result): Result { .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as [number, number, number, number]; const boxRaw = newResult.body[i].boxRaw // update boxRaw .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as [number, number, number, number]; - const keypoints = newResult.body[i].keypoints // update keypoints + const keypoints = (newResult.body[i].keypoints // update keypoints .map((keypoint, j) => ({ score: keypoint.score, part: keypoint.part, - position: { - x: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / bufferedFactor : keypoint.position.x, - y: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / bufferedFactor : keypoint.position.y, - }, - })); + position: [ + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[0] + keypoint.position[0]) / bufferedFactor : keypoint.position[0], + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[1] + keypoint.position[1]) / bufferedFactor : keypoint.position[1], + ], + positionRaw: [ + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[0] + keypoint.positionRaw[0]) / bufferedFactor : keypoint.position[0], + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[1] + keypoint.positionRaw[1]) / bufferedFactor : keypoint.position[1], + ], + }))) as Array<{ score: number, part: string, position: [number, number, number?], positionRaw: [number, number, number?] }>; bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values } } diff --git a/src/movenet/movenet.ts b/src/movenet/movenet.ts index 9d943b38..b78b1160 100644 --- a/src/movenet/movenet.ts +++ b/src/movenet/movenet.ts @@ -9,7 +9,7 @@ import { GraphModel } from '../tfjs/types'; let model: GraphModel; -type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } }; +type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] }; const keypoints: Array = []; let box: [number, number, number, number] = [0, 0, 0, 0]; @@ -58,29 +58,29 @@ export async function predict(image, config): Promise { keypoints.push({ score: Math.round(100 * score) / 100, part: bodyParts[id], - positionRaw: { // normalized to 0..1 - x: kpt[id][1], - y: kpt[id][0], - }, - position: { // normalized to input image size - x: Math.round(image.shape[2] * kpt[id][1]), - y: Math.round(image.shape[1] * kpt[id][0]), - }, + positionRaw: [ // normalized to 0..1 + kpt[id][1], + kpt[id][0], + ], + position: [ // normalized to input image size + Math.round(image.shape[2] * kpt[id][1]), + Math.round(image.shape[1] * kpt[id][0]), + ], }); } } } score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0); - const x = keypoints.map((a) => a.position.x); - const y = keypoints.map((a) => a.position.y); + const x = keypoints.map((a) => a.position[0]); + const y = keypoints.map((a) => a.position[1]); box = [ Math.min(...x), Math.min(...y), Math.max(...x) - Math.min(...x), Math.max(...y) - Math.min(...y), ]; - const xRaw = keypoints.map((a) => a.positionRaw.x); - const yRaw = keypoints.map((a) => a.positionRaw.y); + const xRaw = keypoints.map((a) => a.positionRaw[0]); + const yRaw = keypoints.map((a) => a.positionRaw[1]); boxRaw = [ Math.min(...xRaw), Math.min(...yRaw), diff --git a/src/posenet/utils.ts b/src/posenet/utils.ts index d1aa841e..c8383647 100644 --- a/src/posenet/utils.ts +++ b/src/posenet/utils.ts @@ -1,4 +1,5 @@ import * as kpt from './keypoints'; +import { Body } from '../result'; export function eitherPointDoesntMeetConfidence(a, b, minConfidence) { return (a < minConfidence || b < minConfidence); @@ -29,7 +30,7 @@ export function getBoundingBox(keypoints): [number, number, number, number] { return [coord.minX, coord.minY, coord.maxX - coord.minX, coord.maxY - coord.minY]; } -export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]) { +export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array { const scaleY = height / inputResolutionHeight; const scaleX = width / inputResolutionWidth; const scalePose = (pose, i) => ({ @@ -40,7 +41,8 @@ export function scalePoses(poses, [height, width], [inputResolutionHeight, input keypoints: pose.keypoints.map(({ score, part, position }) => ({ score, part, - position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) }, + position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)], + positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight], })), }); const scaledPoses = poses.map((pose, i) => scalePose(pose, i)); diff --git a/src/result.ts b/src/result.ts index ecc98ab3..f6f63c69 100644 --- a/src/result.ts +++ b/src/result.ts @@ -71,12 +71,12 @@ export interface Face { export interface Body { id: number, score: number, - box: [x: number, y: number, width: number, height: number], - boxRaw: [x: number, y: number, width: number, height: number], + box: [number, number, number, number], + boxRaw: [number, number, number, number], keypoints: Array<{ part: string, - position: { x: number, y: number, z?: number }, - positionRaw?: { x: number, y: number, z?: number }, + position: [number, number, number?], + positionRaw: [number, number, number?], score: number, presence?: number, }>