ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

breaking changes to results.body output properties

pull/134/head
Vladimir Mandic 2021-06-01 06:55:40 -04:00
parent 879574e93e
commit fdc0c8d496
16 changed files with 201080 additions and 4686 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10908
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

67527
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

67539
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

114
dist/human.node-gpu.js vendored
View File

@ -4595,7 +4595,8 @@ function scalePoses(poses2, [height, width], [inputResolutionHeight, inputResolu
keypoints: pose.keypoints.map(({ score: score3, part, position }) => ({ keypoints: pose.keypoints.map(({ score: score3, part, position }) => ({
score: score3, score: score3,
part, part,
position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) } position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)],
positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight]
})) }))
}); });
const scaledPoses = poses2.map((pose, i) => scalePose(pose, i)); const scaledPoses = poses2.map((pose, i) => scalePose(pose, i));
@ -8340,17 +8341,22 @@ async function predict6(image15, config3) {
keypoints3.push({ keypoints3.push({
id: i, id: i,
part: labels2[i], part: labels2[i],
position: { position: [
x: Math.trunc(imgSize.width * points[depth * i + 0] / 255), Math.trunc(imgSize.width * points[depth * i + 0] / 255),
y: Math.trunc(imgSize.height * points[depth * i + 1] / 255), Math.trunc(imgSize.height * points[depth * i + 1] / 255),
z: Math.trunc(points[depth * i + 2]) + 0 Math.trunc(points[depth * i + 2]) + 0
}, ],
positionRaw: [
points[depth * i + 0] / 255,
points[depth * i + 1] / 255,
points[depth * i + 2] + 0
],
score: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 3])))) / 100, score: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 3])))) / 100,
presence: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 4])))) / 100 presence: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 4])))) / 100
}); });
} }
const x = keypoints3.map((a) => a.position.x); const x = keypoints3.map((a) => a.position[0]);
const y = keypoints3.map((a) => a.position.y); const y = keypoints3.map((a) => a.position[1]);
const box6 = [ const box6 = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),
@ -8511,29 +8517,29 @@ async function predict8(image15, config3) {
keypoints2.push({ keypoints2.push({
score: Math.round(100 * score2) / 100, score: Math.round(100 * score2) / 100,
part: bodyParts2[id], part: bodyParts2[id],
positionRaw: { positionRaw: [
x: kpt3[id][1], kpt3[id][1],
y: kpt3[id][0] kpt3[id][0]
}, ],
position: { position: [
x: Math.round(image15.shape[2] * kpt3[id][1]), Math.round(image15.shape[2] * kpt3[id][1]),
y: Math.round(image15.shape[1] * kpt3[id][0]) Math.round(image15.shape[1] * kpt3[id][0])
} ]
}); });
} }
} }
} }
score2 = keypoints2.reduce((prev, curr) => curr.score > prev ? curr.score : prev, 0); score2 = keypoints2.reduce((prev, curr) => curr.score > prev ? curr.score : prev, 0);
const x = keypoints2.map((a) => a.position.x); const x = keypoints2.map((a) => a.position[0]);
const y = keypoints2.map((a) => a.position.y); const y = keypoints2.map((a) => a.position[1]);
box5 = [ box5 = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),
Math.max(...x) - Math.min(...x), Math.max(...x) - Math.min(...x),
Math.max(...y) - Math.min(...y) Math.max(...y) - Math.min(...y)
]; ];
const xRaw = keypoints2.map((a) => a.positionRaw.x); const xRaw = keypoints2.map((a) => a.positionRaw[0]);
const yRaw = keypoints2.map((a) => a.positionRaw.y); const yRaw = keypoints2.map((a) => a.positionRaw[1]);
boxRaw2 = [ boxRaw2 = [
Math.min(...xRaw), Math.min(...xRaw),
Math.min(...yRaw), Math.min(...yRaw),
@ -10044,16 +10050,16 @@ async function body2(inCanvas2, result, drawOptions) {
} }
if (localOptions.drawPoints) { if (localOptions.drawPoints) {
for (let pt = 0; pt < result[i].keypoints.length; pt++) { for (let pt = 0; pt < result[i].keypoints.length; pt++) {
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + 2 * (result[i].keypoints[pt].position.z || 0)}, ${127.5 - 2 * (result[i].keypoints[pt].position.z || 0)}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position[2] ? `rgba(${127.5 + 2 * (result[i].keypoints[pt].position[2] || 0)}, ${127.5 - 2 * (result[i].keypoints[pt].position[2] || 0)}, 255, 0.5)` : localOptions.color;
point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions); point(ctx, result[i].keypoints[pt].position[0], result[i].keypoints[pt].position[1], 0, localOptions);
} }
} }
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
ctx.font = localOptions.font; ctx.font = localOptions.font;
if (result[i].keypoints) { if (result[i].keypoints) {
for (const pt of result[i].keypoints) { for (const pt of result[i].keypoints) {
ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + 2 * pt.position.z}, ${127.5 - 2 * pt.position.z}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && pt.position[2] ? `rgba(${127.5 + 2 * pt.position[2]}, ${127.5 - 2 * pt.position[2]}, 255, 0.5)` : localOptions.color;
ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position.x + 4, pt.position.y + 4); ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position[0] + 4, pt.position[1] + 4);
} }
} }
} }
@ -10063,87 +10069,87 @@ async function body2(inCanvas2, result, drawOptions) {
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder"); part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightShoulder"); part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder"); part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightHip"); part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftHip"); part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftShoulder"); part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
if (points.length === 4) if (points.length === 4)
lines(ctx, points, localOptions); lines(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftHip"); part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftKnee"); part = result[i].keypoints.find((a) => a.part === "leftKnee");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftAnkle"); part = result[i].keypoints.find((a) => a.part === "leftAnkle");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftHeel"); part = result[i].keypoints.find((a) => a.part === "leftHeel");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftFoot"); part = result[i].keypoints.find((a) => a.part === "leftFoot");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightHip"); part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightKnee"); part = result[i].keypoints.find((a) => a.part === "rightKnee");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightAnkle"); part = result[i].keypoints.find((a) => a.part === "rightAnkle");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightHeel"); part = result[i].keypoints.find((a) => a.part === "rightHeel");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightFoot"); part = result[i].keypoints.find((a) => a.part === "rightFoot");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder"); part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftElbow"); part = result[i].keypoints.find((a) => a.part === "leftElbow");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftWrist"); part = result[i].keypoints.find((a) => a.part === "leftWrist");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftPalm"); part = result[i].keypoints.find((a) => a.part === "leftPalm");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder"); part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightElbow"); part = result[i].keypoints.find((a) => a.part === "rightElbow");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightWrist"); part = result[i].keypoints.find((a) => a.part === "rightWrist");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightPalm"); part = result[i].keypoints.find((a) => a.part === "rightPalm");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
} }
} }
@ -10368,10 +10374,14 @@ function calc(newResult) {
const keypoints3 = newResult.body[i].keypoints.map((keypoint, j) => ({ const keypoints3 = newResult.body[i].keypoints.map((keypoint, j) => ({
score: keypoint.score, score: keypoint.score,
part: keypoint.part, part: keypoint.part,
position: { position: [
x: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / bufferedFactor : keypoint.position.x, bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[0] + keypoint.position[0]) / bufferedFactor : keypoint.position[0],
y: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / bufferedFactor : keypoint.position.y bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[1] + keypoint.position[1]) / bufferedFactor : keypoint.position[1]
} ],
positionRaw: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[0] + keypoint.positionRaw[0]) / bufferedFactor : keypoint.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[1] + keypoint.positionRaw[1]) / bufferedFactor : keypoint.position[1]
]
})); }));
bufferedResult.body[i] = { ...newResult.body[i], box: box6, boxRaw: boxRaw3, keypoints: keypoints3 }; bufferedResult.body[i] = { ...newResult.body[i], box: box6, boxRaw: boxRaw3, keypoints: keypoints3 };
} }

114
dist/human.node-wasm.js vendored
View File

@ -4596,7 +4596,8 @@ function scalePoses(poses2, [height, width], [inputResolutionHeight, inputResolu
keypoints: pose.keypoints.map(({ score: score3, part, position }) => ({ keypoints: pose.keypoints.map(({ score: score3, part, position }) => ({
score: score3, score: score3,
part, part,
position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) } position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)],
positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight]
})) }))
}); });
const scaledPoses = poses2.map((pose, i) => scalePose(pose, i)); const scaledPoses = poses2.map((pose, i) => scalePose(pose, i));
@ -8341,17 +8342,22 @@ async function predict6(image15, config3) {
keypoints3.push({ keypoints3.push({
id: i, id: i,
part: labels2[i], part: labels2[i],
position: { position: [
x: Math.trunc(imgSize.width * points[depth * i + 0] / 255), Math.trunc(imgSize.width * points[depth * i + 0] / 255),
y: Math.trunc(imgSize.height * points[depth * i + 1] / 255), Math.trunc(imgSize.height * points[depth * i + 1] / 255),
z: Math.trunc(points[depth * i + 2]) + 0 Math.trunc(points[depth * i + 2]) + 0
}, ],
positionRaw: [
points[depth * i + 0] / 255,
points[depth * i + 1] / 255,
points[depth * i + 2] + 0
],
score: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 3])))) / 100, score: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 3])))) / 100,
presence: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 4])))) / 100 presence: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 4])))) / 100
}); });
} }
const x = keypoints3.map((a) => a.position.x); const x = keypoints3.map((a) => a.position[0]);
const y = keypoints3.map((a) => a.position.y); const y = keypoints3.map((a) => a.position[1]);
const box6 = [ const box6 = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),
@ -8512,29 +8518,29 @@ async function predict8(image15, config3) {
keypoints2.push({ keypoints2.push({
score: Math.round(100 * score2) / 100, score: Math.round(100 * score2) / 100,
part: bodyParts2[id], part: bodyParts2[id],
positionRaw: { positionRaw: [
x: kpt3[id][1], kpt3[id][1],
y: kpt3[id][0] kpt3[id][0]
}, ],
position: { position: [
x: Math.round(image15.shape[2] * kpt3[id][1]), Math.round(image15.shape[2] * kpt3[id][1]),
y: Math.round(image15.shape[1] * kpt3[id][0]) Math.round(image15.shape[1] * kpt3[id][0])
} ]
}); });
} }
} }
} }
score2 = keypoints2.reduce((prev, curr) => curr.score > prev ? curr.score : prev, 0); score2 = keypoints2.reduce((prev, curr) => curr.score > prev ? curr.score : prev, 0);
const x = keypoints2.map((a) => a.position.x); const x = keypoints2.map((a) => a.position[0]);
const y = keypoints2.map((a) => a.position.y); const y = keypoints2.map((a) => a.position[1]);
box5 = [ box5 = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),
Math.max(...x) - Math.min(...x), Math.max(...x) - Math.min(...x),
Math.max(...y) - Math.min(...y) Math.max(...y) - Math.min(...y)
]; ];
const xRaw = keypoints2.map((a) => a.positionRaw.x); const xRaw = keypoints2.map((a) => a.positionRaw[0]);
const yRaw = keypoints2.map((a) => a.positionRaw.y); const yRaw = keypoints2.map((a) => a.positionRaw[1]);
boxRaw2 = [ boxRaw2 = [
Math.min(...xRaw), Math.min(...xRaw),
Math.min(...yRaw), Math.min(...yRaw),
@ -10045,16 +10051,16 @@ async function body2(inCanvas2, result, drawOptions) {
} }
if (localOptions.drawPoints) { if (localOptions.drawPoints) {
for (let pt = 0; pt < result[i].keypoints.length; pt++) { for (let pt = 0; pt < result[i].keypoints.length; pt++) {
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + 2 * (result[i].keypoints[pt].position.z || 0)}, ${127.5 - 2 * (result[i].keypoints[pt].position.z || 0)}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position[2] ? `rgba(${127.5 + 2 * (result[i].keypoints[pt].position[2] || 0)}, ${127.5 - 2 * (result[i].keypoints[pt].position[2] || 0)}, 255, 0.5)` : localOptions.color;
point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions); point(ctx, result[i].keypoints[pt].position[0], result[i].keypoints[pt].position[1], 0, localOptions);
} }
} }
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
ctx.font = localOptions.font; ctx.font = localOptions.font;
if (result[i].keypoints) { if (result[i].keypoints) {
for (const pt of result[i].keypoints) { for (const pt of result[i].keypoints) {
ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + 2 * pt.position.z}, ${127.5 - 2 * pt.position.z}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && pt.position[2] ? `rgba(${127.5 + 2 * pt.position[2]}, ${127.5 - 2 * pt.position[2]}, 255, 0.5)` : localOptions.color;
ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position.x + 4, pt.position.y + 4); ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position[0] + 4, pt.position[1] + 4);
} }
} }
} }
@ -10064,87 +10070,87 @@ async function body2(inCanvas2, result, drawOptions) {
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder"); part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightShoulder"); part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder"); part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightHip"); part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftHip"); part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftShoulder"); part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
if (points.length === 4) if (points.length === 4)
lines(ctx, points, localOptions); lines(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftHip"); part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftKnee"); part = result[i].keypoints.find((a) => a.part === "leftKnee");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftAnkle"); part = result[i].keypoints.find((a) => a.part === "leftAnkle");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftHeel"); part = result[i].keypoints.find((a) => a.part === "leftHeel");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftFoot"); part = result[i].keypoints.find((a) => a.part === "leftFoot");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightHip"); part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightKnee"); part = result[i].keypoints.find((a) => a.part === "rightKnee");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightAnkle"); part = result[i].keypoints.find((a) => a.part === "rightAnkle");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightHeel"); part = result[i].keypoints.find((a) => a.part === "rightHeel");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightFoot"); part = result[i].keypoints.find((a) => a.part === "rightFoot");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder"); part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftElbow"); part = result[i].keypoints.find((a) => a.part === "leftElbow");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftWrist"); part = result[i].keypoints.find((a) => a.part === "leftWrist");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftPalm"); part = result[i].keypoints.find((a) => a.part === "leftPalm");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder"); part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightElbow"); part = result[i].keypoints.find((a) => a.part === "rightElbow");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightWrist"); part = result[i].keypoints.find((a) => a.part === "rightWrist");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightPalm"); part = result[i].keypoints.find((a) => a.part === "rightPalm");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
} }
} }
@ -10369,10 +10375,14 @@ function calc(newResult) {
const keypoints3 = newResult.body[i].keypoints.map((keypoint, j) => ({ const keypoints3 = newResult.body[i].keypoints.map((keypoint, j) => ({
score: keypoint.score, score: keypoint.score,
part: keypoint.part, part: keypoint.part,
position: { position: [
x: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / bufferedFactor : keypoint.position.x, bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[0] + keypoint.position[0]) / bufferedFactor : keypoint.position[0],
y: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / bufferedFactor : keypoint.position.y bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[1] + keypoint.position[1]) / bufferedFactor : keypoint.position[1]
} ],
positionRaw: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[0] + keypoint.positionRaw[0]) / bufferedFactor : keypoint.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[1] + keypoint.positionRaw[1]) / bufferedFactor : keypoint.position[1]
]
})); }));
bufferedResult.body[i] = { ...newResult.body[i], box: box6, boxRaw: boxRaw3, keypoints: keypoints3 }; bufferedResult.body[i] = { ...newResult.body[i], box: box6, boxRaw: boxRaw3, keypoints: keypoints3 };
} }

114
dist/human.node.js vendored
View File

@ -4595,7 +4595,8 @@ function scalePoses(poses2, [height, width], [inputResolutionHeight, inputResolu
keypoints: pose.keypoints.map(({ score: score3, part, position }) => ({ keypoints: pose.keypoints.map(({ score: score3, part, position }) => ({
score: score3, score: score3,
part, part,
position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) } position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)],
positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight]
})) }))
}); });
const scaledPoses = poses2.map((pose, i) => scalePose(pose, i)); const scaledPoses = poses2.map((pose, i) => scalePose(pose, i));
@ -8340,17 +8341,22 @@ async function predict6(image15, config3) {
keypoints3.push({ keypoints3.push({
id: i, id: i,
part: labels2[i], part: labels2[i],
position: { position: [
x: Math.trunc(imgSize.width * points[depth * i + 0] / 255), Math.trunc(imgSize.width * points[depth * i + 0] / 255),
y: Math.trunc(imgSize.height * points[depth * i + 1] / 255), Math.trunc(imgSize.height * points[depth * i + 1] / 255),
z: Math.trunc(points[depth * i + 2]) + 0 Math.trunc(points[depth * i + 2]) + 0
}, ],
positionRaw: [
points[depth * i + 0] / 255,
points[depth * i + 1] / 255,
points[depth * i + 2] + 0
],
score: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 3])))) / 100, score: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 3])))) / 100,
presence: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 4])))) / 100 presence: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 4])))) / 100
}); });
} }
const x = keypoints3.map((a) => a.position.x); const x = keypoints3.map((a) => a.position[0]);
const y = keypoints3.map((a) => a.position.y); const y = keypoints3.map((a) => a.position[1]);
const box6 = [ const box6 = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),
@ -8511,29 +8517,29 @@ async function predict8(image15, config3) {
keypoints2.push({ keypoints2.push({
score: Math.round(100 * score2) / 100, score: Math.round(100 * score2) / 100,
part: bodyParts2[id], part: bodyParts2[id],
positionRaw: { positionRaw: [
x: kpt3[id][1], kpt3[id][1],
y: kpt3[id][0] kpt3[id][0]
}, ],
position: { position: [
x: Math.round(image15.shape[2] * kpt3[id][1]), Math.round(image15.shape[2] * kpt3[id][1]),
y: Math.round(image15.shape[1] * kpt3[id][0]) Math.round(image15.shape[1] * kpt3[id][0])
} ]
}); });
} }
} }
} }
score2 = keypoints2.reduce((prev, curr) => curr.score > prev ? curr.score : prev, 0); score2 = keypoints2.reduce((prev, curr) => curr.score > prev ? curr.score : prev, 0);
const x = keypoints2.map((a) => a.position.x); const x = keypoints2.map((a) => a.position[0]);
const y = keypoints2.map((a) => a.position.y); const y = keypoints2.map((a) => a.position[1]);
box5 = [ box5 = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),
Math.max(...x) - Math.min(...x), Math.max(...x) - Math.min(...x),
Math.max(...y) - Math.min(...y) Math.max(...y) - Math.min(...y)
]; ];
const xRaw = keypoints2.map((a) => a.positionRaw.x); const xRaw = keypoints2.map((a) => a.positionRaw[0]);
const yRaw = keypoints2.map((a) => a.positionRaw.y); const yRaw = keypoints2.map((a) => a.positionRaw[1]);
boxRaw2 = [ boxRaw2 = [
Math.min(...xRaw), Math.min(...xRaw),
Math.min(...yRaw), Math.min(...yRaw),
@ -10044,16 +10050,16 @@ async function body2(inCanvas2, result, drawOptions) {
} }
if (localOptions.drawPoints) { if (localOptions.drawPoints) {
for (let pt = 0; pt < result[i].keypoints.length; pt++) { for (let pt = 0; pt < result[i].keypoints.length; pt++) {
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + 2 * (result[i].keypoints[pt].position.z || 0)}, ${127.5 - 2 * (result[i].keypoints[pt].position.z || 0)}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position[2] ? `rgba(${127.5 + 2 * (result[i].keypoints[pt].position[2] || 0)}, ${127.5 - 2 * (result[i].keypoints[pt].position[2] || 0)}, 255, 0.5)` : localOptions.color;
point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions); point(ctx, result[i].keypoints[pt].position[0], result[i].keypoints[pt].position[1], 0, localOptions);
} }
} }
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
ctx.font = localOptions.font; ctx.font = localOptions.font;
if (result[i].keypoints) { if (result[i].keypoints) {
for (const pt of result[i].keypoints) { for (const pt of result[i].keypoints) {
ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + 2 * pt.position.z}, ${127.5 - 2 * pt.position.z}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && pt.position[2] ? `rgba(${127.5 + 2 * pt.position[2]}, ${127.5 - 2 * pt.position[2]}, 255, 0.5)` : localOptions.color;
ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position.x + 4, pt.position.y + 4); ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position[0] + 4, pt.position[1] + 4);
} }
} }
} }
@ -10063,87 +10069,87 @@ async function body2(inCanvas2, result, drawOptions) {
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder"); part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightShoulder"); part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder"); part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightHip"); part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftHip"); part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftShoulder"); part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
if (points.length === 4) if (points.length === 4)
lines(ctx, points, localOptions); lines(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftHip"); part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftKnee"); part = result[i].keypoints.find((a) => a.part === "leftKnee");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftAnkle"); part = result[i].keypoints.find((a) => a.part === "leftAnkle");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftHeel"); part = result[i].keypoints.find((a) => a.part === "leftHeel");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftFoot"); part = result[i].keypoints.find((a) => a.part === "leftFoot");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightHip"); part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightKnee"); part = result[i].keypoints.find((a) => a.part === "rightKnee");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightAnkle"); part = result[i].keypoints.find((a) => a.part === "rightAnkle");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightHeel"); part = result[i].keypoints.find((a) => a.part === "rightHeel");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightFoot"); part = result[i].keypoints.find((a) => a.part === "rightFoot");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder"); part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftElbow"); part = result[i].keypoints.find((a) => a.part === "leftElbow");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftWrist"); part = result[i].keypoints.find((a) => a.part === "leftWrist");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "leftPalm"); part = result[i].keypoints.find((a) => a.part === "leftPalm");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder"); part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightElbow"); part = result[i].keypoints.find((a) => a.part === "rightElbow");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightWrist"); part = result[i].keypoints.find((a) => a.part === "rightWrist");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === "rightPalm"); part = result[i].keypoints.find((a) => a.part === "rightPalm");
if (part) if (part)
points.push([part.position.x, part.position.y]); points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
} }
} }
@ -10368,10 +10374,14 @@ function calc(newResult) {
const keypoints3 = newResult.body[i].keypoints.map((keypoint, j) => ({ const keypoints3 = newResult.body[i].keypoints.map((keypoint, j) => ({
score: keypoint.score, score: keypoint.score,
part: keypoint.part, part: keypoint.part,
position: { position: [
x: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / bufferedFactor : keypoint.position.x, bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[0] + keypoint.position[0]) / bufferedFactor : keypoint.position[0],
y: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / bufferedFactor : keypoint.position.y bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[1] + keypoint.position[1]) / bufferedFactor : keypoint.position[1]
} ],
positionRaw: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[0] + keypoint.positionRaw[0]) / bufferedFactor : keypoint.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[1] + keypoint.positionRaw[1]) / bufferedFactor : keypoint.position[1]
]
})); }));
bufferedResult.body[i] = { ...newResult.body[i], box: box6, boxRaw: boxRaw3, keypoints: keypoints3 }; bufferedResult.body[i] = { ...newResult.body[i], box: box6, boxRaw: boxRaw3, keypoints: keypoints3 };
} }

59233
dist/tfjs.esm.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/tfjs.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

21
src/blazepose/blazepose.ts
View File

@ -35,24 +35,29 @@ export async function predict(image, config): Promise<Body[]> {
const points = resT.find((t) => (t.size === 195 || t.size === 155))?.dataSync() || []; // order of output tensors may change between models, full has 195 and upper has 155 items const points = resT.find((t) => (t.size === 195 || t.size === 155))?.dataSync() || []; // order of output tensors may change between models, full has 195 and upper has 155 items
resT.forEach((t) => t.dispose()); resT.forEach((t) => t.dispose());
normalize.dispose(); normalize.dispose();
const keypoints: Array<{ id, part, position: { x, y, z }, score, presence }> = []; const keypoints: Array<{ id, part, position: [number, number, number], positionRaw: [number, number, number], score, presence }> = [];
const labels = points?.length === 195 ? annotations.full : annotations.upper; // full model has 39 keypoints, upper has 31 keypoints const labels = points?.length === 195 ? annotations.full : annotations.upper; // full model has 39 keypoints, upper has 31 keypoints
const depth = 5; // each points has x,y,z,visibility,presence const depth = 5; // each points has x,y,z,visibility,presence
for (let i = 0; i < points.length / depth; i++) { for (let i = 0; i < points.length / depth; i++) {
keypoints.push({ keypoints.push({
id: i, id: i,
part: labels[i], part: labels[i],
position: { position: [
x: Math.trunc(imgSize.width * points[depth * i + 0] / 255), // return normalized x value istead of 0..255 Math.trunc(imgSize.width * points[depth * i + 0] / 255), // return normalized x value istead of 0..255
y: Math.trunc(imgSize.height * points[depth * i + 1] / 255), // return normalized y value istead of 0..255 Math.trunc(imgSize.height * points[depth * i + 1] / 255), // return normalized y value istead of 0..255
z: Math.trunc(points[depth * i + 2]) + 0, // fix negative zero Math.trunc(points[depth * i + 2]) + 0, // fix negative zero
}, ],
positionRaw: [
points[depth * i + 0] / 255, // return x value normalized to 0..1
points[depth * i + 1] / 255, // return y value normalized to 0..1
points[depth * i + 2] + 0, // fix negative zero
],
score: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 3])))) / 100, // reverse sigmoid value score: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 3])))) / 100, // reverse sigmoid value
presence: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 4])))) / 100, // reverse sigmoid value presence: (100 - Math.trunc(100 / (1 + Math.exp(points[depth * i + 4])))) / 100, // reverse sigmoid value
}); });
} }
const x = keypoints.map((a) => a.position.x); const x = keypoints.map((a) => a.position[0]);
const y = keypoints.map((a) => a.position.y); const y = keypoints.map((a) => a.position[1]);
const box: [number, number, number, number] = [ const box: [number, number, number, number] = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),

56
src/draw/draw.ts
View File

@ -293,16 +293,16 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, dra
} }
if (localOptions.drawPoints) { if (localOptions.drawPoints) {
for (let pt = 0; pt < result[i].keypoints.length; pt++) { for (let pt = 0; pt < result[i].keypoints.length; pt++) {
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position.z || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position.z || 0))}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position[2] ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position[2] || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position[2] || 0))}, 255, 0.5)` : localOptions.color;
point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions); point(ctx, result[i].keypoints[pt].position[0], result[i].keypoints[pt].position[1], 0, localOptions);
} }
} }
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
ctx.font = localOptions.font; ctx.font = localOptions.font;
if (result[i].keypoints) { if (result[i].keypoints) {
for (const pt of result[i].keypoints) { for (const pt of result[i].keypoints) {
ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && pt.position[2] ? `rgba(${127.5 + (2 * pt.position[2])}, ${127.5 - (2 * pt.position[2])}, 255, 0.5)` : localOptions.color;
ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position.x + 4, pt.position.y + 4); ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position[0] + 4, pt.position[1] + 4);
} }
} }
} }
@ -312,68 +312,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, dra
// shoulder line // shoulder line
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// torso main // torso main
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightHip'); part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftHip'); part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
// leg left // leg left
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftHip'); part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftKnee'); part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftHeel'); part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftFoot'); part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// leg right // leg right
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightHip'); part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightKnee'); part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightHeel'); part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightFoot'); part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// arm left // arm left
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftElbow'); part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftWrist'); part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'leftPalm'); part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// arm right // arm right
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightElbow'); part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightWrist'); part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
part = result[i].keypoints.find((a) => a.part === 'rightPalm'); part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part) points.push([part.position.x, part.position.y]); if (part) points.push([part.position[0], part.position[1]]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// draw all // draw all
} }

16
src/interpolate.ts
View File

@ -22,15 +22,19 @@ export function calc(newResult: Result): Result {
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as [number, number, number, number]; .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as [number, number, number, number];
const boxRaw = newResult.body[i].boxRaw // update boxRaw const boxRaw = newResult.body[i].boxRaw // update boxRaw
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as [number, number, number, number]; .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as [number, number, number, number];
const keypoints = newResult.body[i].keypoints // update keypoints const keypoints = (newResult.body[i].keypoints // update keypoints
.map((keypoint, j) => ({ .map((keypoint, j) => ({
score: keypoint.score, score: keypoint.score,
part: keypoint.part, part: keypoint.part,
position: { position: [
x: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / bufferedFactor : keypoint.position.x, bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[0] + keypoint.position[0]) / bufferedFactor : keypoint.position[0],
y: bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / bufferedFactor : keypoint.position.y, bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[1] + keypoint.position[1]) / bufferedFactor : keypoint.position[1],
}, ],
})); positionRaw: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[0] + keypoint.positionRaw[0]) / bufferedFactor : keypoint.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[1] + keypoint.positionRaw[1]) / bufferedFactor : keypoint.position[1],
],
}))) as Array<{ score: number, part: string, position: [number, number, number?], positionRaw: [number, number, number?] }>;
bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values
} }
} }

26
src/movenet/movenet.ts
View File

@ -9,7 +9,7 @@ import { GraphModel } from '../tfjs/types';
let model: GraphModel; let model: GraphModel;
type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } }; type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] };
const keypoints: Array<Keypoints> = []; const keypoints: Array<Keypoints> = [];
let box: [number, number, number, number] = [0, 0, 0, 0]; let box: [number, number, number, number] = [0, 0, 0, 0];
@ -58,29 +58,29 @@ export async function predict(image, config): Promise<Body[]> {
keypoints.push({ keypoints.push({
score: Math.round(100 * score) / 100, score: Math.round(100 * score) / 100,
part: bodyParts[id], part: bodyParts[id],
positionRaw: { // normalized to 0..1 positionRaw: [ // normalized to 0..1
x: kpt[id][1], kpt[id][1],
y: kpt[id][0], kpt[id][0],
}, ],
position: { // normalized to input image size position: [ // normalized to input image size
x: Math.round(image.shape[2] * kpt[id][1]), Math.round(image.shape[2] * kpt[id][1]),
y: Math.round(image.shape[1] * kpt[id][0]), Math.round(image.shape[1] * kpt[id][0]),
}, ],
}); });
} }
} }
} }
score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0); score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
const x = keypoints.map((a) => a.position.x); const x = keypoints.map((a) => a.position[0]);
const y = keypoints.map((a) => a.position.y); const y = keypoints.map((a) => a.position[1]);
box = [ box = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),
Math.max(...x) - Math.min(...x), Math.max(...x) - Math.min(...x),
Math.max(...y) - Math.min(...y), Math.max(...y) - Math.min(...y),
]; ];
const xRaw = keypoints.map((a) => a.positionRaw.x); const xRaw = keypoints.map((a) => a.positionRaw[0]);
const yRaw = keypoints.map((a) => a.positionRaw.y); const yRaw = keypoints.map((a) => a.positionRaw[1]);
boxRaw = [ boxRaw = [
Math.min(...xRaw), Math.min(...xRaw),
Math.min(...yRaw), Math.min(...yRaw),

6
src/posenet/utils.ts
View File

@ -1,4 +1,5 @@
import * as kpt from './keypoints'; import * as kpt from './keypoints';
import { Body } from '../result';
export function eitherPointDoesntMeetConfidence(a, b, minConfidence) { export function eitherPointDoesntMeetConfidence(a, b, minConfidence) {
return (a < minConfidence || b < minConfidence); return (a < minConfidence || b < minConfidence);
@ -29,7 +30,7 @@ export function getBoundingBox(keypoints): [number, number, number, number] {
return [coord.minX, coord.minY, coord.maxX - coord.minX, coord.maxY - coord.minY]; return [coord.minX, coord.minY, coord.maxX - coord.minX, coord.maxY - coord.minY];
} }
export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]) { export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array<Body> {
const scaleY = height / inputResolutionHeight; const scaleY = height / inputResolutionHeight;
const scaleX = width / inputResolutionWidth; const scaleX = width / inputResolutionWidth;
const scalePose = (pose, i) => ({ const scalePose = (pose, i) => ({
@ -40,7 +41,8 @@ export function scalePoses(poses, [height, width], [inputResolutionHeight, input
keypoints: pose.keypoints.map(({ score, part, position }) => ({ keypoints: pose.keypoints.map(({ score, part, position }) => ({
score, score,
part, part,
position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) }, position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)],
positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight],
})), })),
}); });
const scaledPoses = poses.map((pose, i) => scalePose(pose, i)); const scaledPoses = poses.map((pose, i) => scalePose(pose, i));

8
src/result.ts
View File

@ -71,12 +71,12 @@ export interface Face {
export interface Body { export interface Body {
id: number, id: number,
score: number, score: number,
box: [x: number, y: number, width: number, height: number], box: [number, number, number, number],
boxRaw: [x: number, y: number, width: number, height: number], boxRaw: [number, number, number, number],
keypoints: Array<{ keypoints: Array<{
part: string, part: string,
position: { x: number, y: number, z?: number }, position: [number, number, number?],
positionRaw?: { x: number, y: number, z?: number }, positionRaw: [number, number, number?],
score: number, score: number,
presence?: number, presence?: number,
}> }>