ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

breaking changes to results.body output properties

pull/280/head
Vladimir Mandic 2021-06-01 06:55:40 -04:00
parent d71bf8bb2f
commit efc7e530df
5 changed files with 59 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/draw/draw.ts
View File

@ -293,16 +293,16 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, dra
} }
if (localOptions.drawPoints) { if (localOptions.drawPoints) {
for (let pt = 0; pt < result[i].keypoints.length; pt++) { for (let pt = 0; pt < result[i].keypoints.length; pt++) {
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position.z || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position.z || 0))}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position[2] ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position[2] || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position[2] || 0))}, 255, 0.5)` : localOptions.color;
point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions); point(ctx, result[i].keypoints[pt].position[0], result[i].keypoints[pt].position[1], 0, localOptions);
} }
} }
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
ctx.font = localOptions.font; ctx.font = localOptions.font;
if (result[i].keypoints) { if (result[i].keypoints) {
for (const pt of result[i].keypoints) { for (const pt of result[i].keypoints) {
ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && pt.position[2] ? `rgba(${127.5 + (2 * pt.position[2])}, ${127.5 - (2 * pt.position[2])}, 255, 0.5)` : localOptions.color;
ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position.x + 4, pt.position.y + 4); ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position[0] + 4, pt.position[1] + 4);
} }
} }
} }
@ -312,68 +312,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, dra
// shoulder line // shoulder line
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// torso main // torso main
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightHip'); part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftHip'); part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
// leg left // leg left
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftHip'); part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftKnee'); part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftHeel'); part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftFoot'); part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// leg right // leg right
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightHip'); part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightKnee'); part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightHeel'); part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightFoot'); part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// arm left // arm left
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftElbow'); part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftWrist'); part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftPalm'); part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// arm right // arm right
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightElbow'); part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightWrist'); part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightPalm'); part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// draw all // draw all
} }

16
src/interpolate.ts
View File

@ -22,15 +22,19 @@ export function calc(newResult: Result): Result {
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as [number, number, number, number]; .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as [number, number, number, number];
const boxRaw = newResult.body[i].boxRaw // update boxRaw const boxRaw = newResult.body[i].boxRaw // update boxRaw
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as [number, number, number, number]; .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as [number, number, number, number];
const keypoints = newResult.body[i].keypoints // update keypoints const keypoints = (newResult.body[i].keypoints // update keypoints
.map((keypoint, j) => ({ .map((keypoint, j) => ({
score: keypoint.score, score: keypoint.score,
part: keypoint.part, part: keypoint.part,
position: { position: [
x: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / bufferedFactor : keypoint.position.x, bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[0] + keypoint.position[0]) / bufferedFactor : keypoint.position[0],
y: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / bufferedFactor : keypoint.position.y, bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[1] + keypoint.position[1]) / bufferedFactor : keypoint.position[1],
}, ],
})); positionRaw: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[0] + keypoint.positionRaw[0]) / bufferedFactor : keypoint.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[1] + keypoint.positionRaw[1]) / bufferedFactor : keypoint.position[1],
],
}))) as Array<{ score: number, part: string, position: [number, number, number?], positionRaw: [number, number, number?] }>;
bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values
} }
} }

26
src/movenet/movenet.ts
View File

@ -9,7 +9,7 @@ import { GraphModel } from '../tfjs/types';
let model: GraphModel; let model: GraphModel;
type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } }; type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] };
const keypoints: Array<Keypoints> = []; const keypoints: Array<Keypoints> = [];
let box: [number, number, number, number] = [0, 0, 0, 0]; let box: [number, number, number, number] = [0, 0, 0, 0];
@ -58,29 +58,29 @@ export async function predict(image, config): Promise<Body[]> {
keypoints.push({ keypoints.push({
score: Math.round(100 * score) / 100, score: Math.round(100 * score) / 100,
part: bodyParts[id], part: bodyParts[id],
positionRaw: { // normalized to 0..1 positionRaw: [ // normalized to 0..1
x: kpt[id][1], kpt[id][1],
y: kpt[id][0], kpt[id][0],
}, ],
position: { // normalized to input image size position: [ // normalized to input image size
x: Math.round(image.shape[2] * kpt[id][1]), Math.round(image.shape[2] * kpt[id][1]),
y: Math.round(image.shape[1] * kpt[id][0]), Math.round(image.shape[1] * kpt[id][0]),
}, ],
}); });
} }
} }
} }
score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0); score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
const x = keypoints.map((a) => a.position.x); const x = keypoints.map((a) => a.position[0]);
const y = keypoints.map((a) => a.position.y); const y = keypoints.map((a) => a.position[1]);
box = [ box = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),
Math.max(...x) - Math.min(...x), Math.max(...x) - Math.min(...x),
Math.max(...y) - Math.min(...y), Math.max(...y) - Math.min(...y),
]; ];
const xRaw = keypoints.map((a) => a.positionRaw.x); const xRaw = keypoints.map((a) => a.positionRaw[0]);
const yRaw = keypoints.map((a) => a.positionRaw.y); const yRaw = keypoints.map((a) => a.positionRaw[1]);
boxRaw = [ boxRaw = [
Math.min(...xRaw), Math.min(...xRaw),
Math.min(...yRaw), Math.min(...yRaw),

6
src/posenet/utils.ts
View File

@ -1,4 +1,5 @@
import * as kpt from './keypoints'; import * as kpt from './keypoints';
import { Body } from '../result';
export function eitherPointDoesntMeetConfidence(a, b, minConfidence) { export function eitherPointDoesntMeetConfidence(a, b, minConfidence) {
return (a < minConfidence || b < minConfidence); return (a < minConfidence || b < minConfidence);
@ -29,7 +30,7 @@ export function getBoundingBox(keypoints): [number, number, number, number] {
return [coord.minX, coord.minY, coord.maxX - coord.minX, coord.maxY - coord.minY]; return [coord.minX, coord.minY, coord.maxX - coord.minX, coord.maxY - coord.minY];
} }
export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]) { export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array<Body> {
const scaleY = height / inputResolutionHeight; const scaleY = height / inputResolutionHeight;
const scaleX = width / inputResolutionWidth; const scaleX = width / inputResolutionWidth;
const scalePose = (pose, i) => ({ const scalePose = (pose, i) => ({
@ -40,7 +41,8 @@ export function scalePoses(poses, [height, width], [inputResolutionHeight, input
keypoints: pose.keypoints.map(({ score, part, position }) => ({ keypoints: pose.keypoints.map(({ score, part, position }) => ({
score, score,
part, part,
position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) }, position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)],
positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight],
})), })),
}); });
const scaledPoses = poses.map((pose, i) => scalePose(pose, i)); const scaledPoses = poses.map((pose, i) => scalePose(pose, i));

8
src/result.ts
View File

@ -71,12 +71,12 @@ export interface Face {
export interface Body { export interface Body {
id: number, id: number,
score: number, score: number,
box: [x: number, y: number, width: number, height: number], box: [number, number, number, number],
boxRaw: [x: number, y: number, width: number, height: number], boxRaw: [number, number, number, number],
keypoints: Array<{ keypoints: Array<{
part: string, part: string,
position: { x: number, y: number, z?: number }, position: [number, number, number?],
positionRaw?: { x: number, y: number, z?: number }, positionRaw: [number, number, number?],
score: number, score: number,
presence?: number, presence?: number,
}> }>