ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

breaking changes to results.body output properties

pull/280/head
Vladimir Mandic 2021-06-01 06:55:40 -04:00
parent d71bf8bb2f
commit efc7e530df
5 changed files with 59 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/draw/draw.ts
View File

@ -293,16 +293,16 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, dra
}
if (localOptions.drawPoints) {
for (let pt = 0; pt < result[i].keypoints.length; pt++) {
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position.z || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position.z || 0))}, 255, 0.5)` : localOptions.color;
point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions);
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position[2] ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position[2] || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position[2] || 0))}, 255, 0.5)` : localOptions.color;
point(ctx, result[i].keypoints[pt].position[0], result[i].keypoints[pt].position[1], 0, localOptions);
}
}
if (localOptions.drawLabels) {
ctx.font = localOptions.font;
if (result[i].keypoints) {
for (const pt of result[i].keypoints) {
ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : localOptions.color;
ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position.x + 4, pt.position.y + 4);
ctx.fillStyle = localOptions.useDepth && pt.position[2] ? `rgba(${127.5 + (2 * pt.position[2])}, ${127.5 - (2 * pt.position[2])}, 255, 0.5)` : localOptions.color;
ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position[0] + 4, pt.position[1] + 4);
}
}
}
@ -312,68 +312,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, dra
// shoulder line
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions);
// torso main
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
// leg left
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions);
// leg right
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions);
// arm left
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions);
// arm right
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions);
// draw all
}

16
src/interpolate.ts
View File

@ -22,15 +22,19 @@ export function calc(newResult: Result): Result {
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as [number, number, number, number];
const boxRaw = newResult.body[i].boxRaw // update boxRaw
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as [number, number, number, number];
const keypoints = newResult.body[i].keypoints // update keypoints
const keypoints = (newResult.body[i].keypoints // update keypoints
.map((keypoint, j) => ({
score: keypoint.score,
part: keypoint.part,
position: {
x: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / bufferedFactor : keypoint.position.x,
y: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / bufferedFactor : keypoint.position.y,
},
}));
position: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[0] + keypoint.position[0]) / bufferedFactor : keypoint.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[1] + keypoint.position[1]) / bufferedFactor : keypoint.position[1],
],
positionRaw: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[0] + keypoint.positionRaw[0]) / bufferedFactor : keypoint.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[1] + keypoint.positionRaw[1]) / bufferedFactor : keypoint.position[1],
],
}))) as Array<{ score: number, part: string, position: [number, number, number?], positionRaw: [number, number, number?] }>;
bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values
}
}

26
src/movenet/movenet.ts
View File

@ -9,7 +9,7 @@ import { GraphModel } from '../tfjs/types';
let model: GraphModel;
type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } };
type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] };
const keypoints: Array<Keypoints> = [];
let box: [number, number, number, number] = [0, 0, 0, 0];
@ -58,29 +58,29 @@ export async function predict(image, config): Promise<Body[]> {
keypoints.push({
score: Math.round(100 * score) / 100,
part: bodyParts[id],
positionRaw: { // normalized to 0..1
x: kpt[id][1],
y: kpt[id][0],
},
position: { // normalized to input image size
x: Math.round(image.shape[2] * kpt[id][1]),
y: Math.round(image.shape[1] * kpt[id][0]),
},
positionRaw: [ // normalized to 0..1
kpt[id][1],
kpt[id][0],
],
position: [ // normalized to input image size
Math.round(image.shape[2] * kpt[id][1]),
Math.round(image.shape[1] * kpt[id][0]),
],
});
}
}
}
score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
const x = keypoints.map((a) => a.position.x);
const y = keypoints.map((a) => a.position.y);
const x = keypoints.map((a) => a.position[0]);
const y = keypoints.map((a) => a.position[1]);
box = [
Math.min(...x),
Math.min(...y),
Math.max(...x) - Math.min(...x),
Math.max(...y) - Math.min(...y),
];
const xRaw = keypoints.map((a) => a.positionRaw.x);
const yRaw = keypoints.map((a) => a.positionRaw.y);
const xRaw = keypoints.map((a) => a.positionRaw[0]);
const yRaw = keypoints.map((a) => a.positionRaw[1]);
boxRaw = [
Math.min(...xRaw),
Math.min(...yRaw),

6
src/posenet/utils.ts
View File

@ -1,4 +1,5 @@
import * as kpt from './keypoints';
import { Body } from '../result';
export function eitherPointDoesntMeetConfidence(a, b, minConfidence) {
return (a < minConfidence || b < minConfidence);
@ -29,7 +30,7 @@ export function getBoundingBox(keypoints): [number, number, number, number] {
return [coord.minX, coord.minY, coord.maxX - coord.minX, coord.maxY - coord.minY];
}
export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]) {
export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array<Body> {
const scaleY = height / inputResolutionHeight;
const scaleX = width / inputResolutionWidth;
const scalePose = (pose, i) => ({
@ -40,7 +41,8 @@ export function scalePoses(poses, [height, width], [inputResolutionHeight, input
keypoints: pose.keypoints.map(({ score, part, position }) => ({
score,
part,
position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) },
position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)],
positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight],
})),
});
const scaledPoses = poses.map((pose, i) => scalePose(pose, i));

8
src/result.ts
View File

@ -71,12 +71,12 @@ export interface Face {
export interface Body {
id: number,
score: number,
box: [x: number, y: number, width: number, height: number],
boxRaw: [x: number, y: number, width: number, height: number],
box: [number, number, number, number],
boxRaw: [number, number, number, number],
keypoints: Array<{
part: string,
position: { x: number, y: number, z?: number },
positionRaw?: { x: number, y: number, z?: number },
position: [number, number, number?],
positionRaw: [number, number, number?],
score: number,
presence?: number,
}>