ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switch posenet weights

pull/134/head
Vladimir Mandic 2021-05-04 20:46:33 -04:00
parent d7695638cb
commit 08d4809ffb
19 changed files with 994 additions and 941 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -9,6 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog
### **HEAD -> main** 2021/05/04 mandic00@live.com
### **1.8.2** 2021/05/04 mandic00@live.com

16
demo/index.js
View File

@ -9,17 +9,17 @@ import webRTC from './helpers/webrtc.js';
let human;
const userConfig = {
warmup: 'none',
warmup: 'full',
/*
backend: 'humangl',
async: false,
profile: false,
videoOptimized: true,
backend: 'webgl',
async: true,
videoOptimized: false,
filter: {
enabled: false,
flip: false,
},
face: { enabled: true,
face: { enabled: false,
detector: { return: false },
mesh: { enabled: true },
iris: { enabled: true },
@ -28,7 +28,7 @@ const userConfig = {
},
hand: { enabled: false },
gesture: { enabled: false },
body: { enabled: false, modelPath: 'posenet.json' },
body: { enabled: true, modelPath: 'posenet.json' },
// body: { enabled: true, modelPath: 'blazepose.json' },
// object: { enabled: true },
*/
@ -58,7 +58,7 @@ const ui = {
detectFPS: [], // internal, holds fps values for detection performance
drawFPS: [], // internal, holds fps values for draw performance
buffered: true, // should output be buffered between frames
drawWarmup: false, // debug only, should warmup image processing be displayed on startup
drawWarmup: true, // debug only, should warmup image processing be displayed on startup
drawThread: null, // internl, perform draw operations in a separate thread
detectThread: null, // internl, perform detect operations in a separate thread
framesDraw: 0, // internal, statistics on frames drawn

10
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

348
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

348
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

277
dist/human.node-gpu.js vendored
View File

@ -4533,12 +4533,12 @@ function getOffsetPoint(y, x, keypoint, offsets) {
x: offsets.get(y, x, keypoint + count)
};
}
function getImageCoords(part, outputStride, offsets) {
function getImageCoords(part, outputStride2, offsets) {
const {heatmapY, heatmapX, id: keypoint} = part;
const {y, x} = getOffsetPoint(heatmapY, heatmapX, keypoint, offsets);
return {
x: part.heatmapX * outputStride + x,
y: part.heatmapY * outputStride + y
x: part.heatmapX * outputStride2 + x,
y: part.heatmapY * outputStride2 + y
};
}
function clamp(a, min, max) {
@ -4559,9 +4559,9 @@ function addVectors(a, b) {
// src/posenet/poses.ts
var localMaximumRadius = 1;
var defaultOutputStride = 16;
var squaredNmsRadius = 20 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
var outputStride = 16;
var squaredNmsRadius = 50 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, displacements, offsetRefineStep = 2) {
const getDisplacement = (point2) => ({
y: displacements.get(point2.y, point2.x, edgeId),
x: displacements.get(point2.y, point2.x, displacements.shape[2] / 2 + edgeId)
@ -4587,32 +4587,31 @@ function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scor
const score = scoresBuffer.get(targetKeyPointIndices.y, targetKeyPointIndices.x, targetKeypointId);
return {position: targetKeypoint, part: partNames[targetKeypointId], score};
}
function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) {
function decodePose(root, scores, offsets, displacementsFwd, displacementsBwd) {
const parentChildrenTuples = poseChain.map(([parentJoinName, childJoinName]) => [partIds[parentJoinName], partIds[childJoinName]]);
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
const numParts = scores.shape[2];
const numEdges = parentToChildEdges.length;
const instanceKeypoints = new Array(numParts);
const {part: rootPart, score: rootScore} = root;
const rootPoint = getImageCoords(rootPart, outputStride, offsets);
instanceKeypoints[rootPart.id] = {
score: rootScore,
part: partNames[rootPart.id],
const rootPoint = getImageCoords(root.part, outputStride, offsets);
instanceKeypoints[root.part.id] = {
score: root.score,
part: partNames[root.part.id],
position: rootPoint
};
for (let edge = numEdges - 1; edge >= 0; --edge) {
const sourceKeypointId = parentToChildEdges[edge];
const targetKeypointId = childToParentEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsBwd);
}
}
for (let edge = 0; edge < numEdges; ++edge) {
const sourceKeypointId = childToParentEdges[edge];
const targetKeypointId = parentToChildEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsFwd);
}
}
return instanceKeypoints;
@ -4671,11 +4670,11 @@ function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacemen
const queue = buildPartWithScoreQueue(minConfidence, scoresBuffer);
while (poses2.length < maxDetected && !queue.empty()) {
const root = queue.dequeue();
const rootImageCoords = getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
const rootImageCoords = getImageCoords(root.part, outputStride, offsetsBuffer);
if (withinRadius(poses2, rootImageCoords, root.part.id))
continue;
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
const keypoints = allKeypoints.filter((a) => a.score > minConfidence);
let keypoints = decodePose(root, scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer);
keypoints = keypoints.filter((a) => a.score > minConfidence);
const score = getInstanceScore(poses2, keypoints);
const box4 = getBoundingBox(keypoints);
if (score > minConfidence)
@ -18205,105 +18204,6 @@ __export(draw_exports, {
object: () => object,
options: () => options
});
// src/config.ts
var config2 = {
backend: "webgl",
modelBasePath: "../models/",
wasmPath: "../assets/",
debug: true,
async: true,
videoOptimized: true,
warmup: "full",
filter: {
enabled: true,
width: 0,
height: 0,
flip: false,
return: true,
brightness: 0,
contrast: 0,
sharpness: 0,
blur: 0,
saturation: 0,
hue: 0,
negative: false,
sepia: false,
vintage: false,
kodachrome: false,
technicolor: false,
polaroid: false,
pixelate: 0
},
gesture: {
enabled: true
},
face: {
enabled: true,
detector: {
modelPath: "blazeface.json",
rotation: false,
maxDetected: 10,
skipFrames: 21,
skipInitial: false,
minConfidence: 0.2,
iouThreshold: 0.1,
return: false
},
mesh: {
enabled: true,
modelPath: "facemesh.json"
},
iris: {
enabled: true,
modelPath: "iris.json"
},
description: {
enabled: true,
modelPath: "faceres.json",
skipFrames: 31,
minConfidence: 0.1
},
emotion: {
enabled: true,
minConfidence: 0.1,
skipFrames: 32,
modelPath: "emotion.json"
}
},
body: {
enabled: true,
modelPath: "posenet.json",
maxDetected: 1,
minConfidence: 0.2
},
hand: {
enabled: true,
rotation: false,
skipFrames: 12,
skipInitial: false,
minConfidence: 0.1,
iouThreshold: 0.1,
maxDetected: 1,
landmarks: true,
detector: {
modelPath: "handdetect.json"
},
skeleton: {
modelPath: "handskeleton.json"
}
},
object: {
enabled: false,
modelPath: "nanodet.json",
minConfidence: 0.2,
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 41
}
};
// src/draw/draw.ts
var options = {
color: "rgba(173, 216, 230, 0.3)",
labelColor: "rgba(173, 216, 230, 1)",
@ -18561,87 +18461,87 @@ async function body2(inCanvas2, result, drawOptions) {
const points = [];
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
if (points.length === 4)
lines(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftKnee");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftAnkle");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftHeel");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftFoot");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightKnee");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightAnkle");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightHeel");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightFoot");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftElbow");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftWrist");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftPalm");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightElbow");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightWrist");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightPalm");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
}
@ -18791,6 +18691,103 @@ async function all(inCanvas2, result, drawOptions) {
object(inCanvas2, result.object, localOptions);
}
// src/config.ts
var config2 = {
backend: "webgl",
modelBasePath: "../models/",
wasmPath: "../assets/",
debug: true,
async: true,
videoOptimized: true,
warmup: "full",
filter: {
enabled: true,
width: 0,
height: 0,
flip: false,
return: true,
brightness: 0,
contrast: 0,
sharpness: 0,
blur: 0,
saturation: 0,
hue: 0,
negative: false,
sepia: false,
vintage: false,
kodachrome: false,
technicolor: false,
polaroid: false,
pixelate: 0
},
gesture: {
enabled: true
},
face: {
enabled: true,
detector: {
modelPath: "blazeface.json",
rotation: false,
maxDetected: 10,
skipFrames: 21,
skipInitial: false,
minConfidence: 0.2,
iouThreshold: 0.1,
return: false
},
mesh: {
enabled: true,
modelPath: "facemesh.json"
},
iris: {
enabled: true,
modelPath: "iris.json"
},
description: {
enabled: true,
modelPath: "faceres.json",
skipFrames: 31,
minConfidence: 0.1
},
emotion: {
enabled: true,
minConfidence: 0.1,
skipFrames: 32,
modelPath: "emotion.json"
}
},
body: {
enabled: true,
modelPath: "posenet.json",
maxDetected: 1,
minConfidence: 0.1
},
hand: {
enabled: true,
rotation: false,
skipFrames: 12,
skipInitial: false,
minConfidence: 0.1,
iouThreshold: 0.1,
maxDetected: 1,
landmarks: true,
detector: {
modelPath: "handdetect.json"
},
skeleton: {
modelPath: "handskeleton.json"
}
},
object: {
enabled: false,
modelPath: "nanodet.json",
minConfidence: 0.2,
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 41
}
};
// src/sample.ts
var face3 = `
/9j/4AAQSkZJRgABAQEAYABgAAD/4QBoRXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUA

277
dist/human.node-wasm.js vendored
View File

@ -4534,12 +4534,12 @@ function getOffsetPoint(y, x, keypoint, offsets) {
x: offsets.get(y, x, keypoint + count)
};
}
function getImageCoords(part, outputStride, offsets) {
function getImageCoords(part, outputStride2, offsets) {
const {heatmapY, heatmapX, id: keypoint} = part;
const {y, x} = getOffsetPoint(heatmapY, heatmapX, keypoint, offsets);
return {
x: part.heatmapX * outputStride + x,
y: part.heatmapY * outputStride + y
x: part.heatmapX * outputStride2 + x,
y: part.heatmapY * outputStride2 + y
};
}
function clamp(a, min, max) {
@ -4560,9 +4560,9 @@ function addVectors(a, b) {
// src/posenet/poses.ts
var localMaximumRadius = 1;
var defaultOutputStride = 16;
var squaredNmsRadius = 20 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
var outputStride = 16;
var squaredNmsRadius = 50 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, displacements, offsetRefineStep = 2) {
const getDisplacement = (point2) => ({
y: displacements.get(point2.y, point2.x, edgeId),
x: displacements.get(point2.y, point2.x, displacements.shape[2] / 2 + edgeId)
@ -4588,32 +4588,31 @@ function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scor
const score = scoresBuffer.get(targetKeyPointIndices.y, targetKeyPointIndices.x, targetKeypointId);
return {position: targetKeypoint, part: partNames[targetKeypointId], score};
}
function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) {
function decodePose(root, scores, offsets, displacementsFwd, displacementsBwd) {
const parentChildrenTuples = poseChain.map(([parentJoinName, childJoinName]) => [partIds[parentJoinName], partIds[childJoinName]]);
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
const numParts = scores.shape[2];
const numEdges = parentToChildEdges.length;
const instanceKeypoints = new Array(numParts);
const {part: rootPart, score: rootScore} = root;
const rootPoint = getImageCoords(rootPart, outputStride, offsets);
instanceKeypoints[rootPart.id] = {
score: rootScore,
part: partNames[rootPart.id],
const rootPoint = getImageCoords(root.part, outputStride, offsets);
instanceKeypoints[root.part.id] = {
score: root.score,
part: partNames[root.part.id],
position: rootPoint
};
for (let edge = numEdges - 1; edge >= 0; --edge) {
const sourceKeypointId = parentToChildEdges[edge];
const targetKeypointId = childToParentEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsBwd);
}
}
for (let edge = 0; edge < numEdges; ++edge) {
const sourceKeypointId = childToParentEdges[edge];
const targetKeypointId = parentToChildEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsFwd);
}
}
return instanceKeypoints;
@ -4672,11 +4671,11 @@ function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacemen
const queue = buildPartWithScoreQueue(minConfidence, scoresBuffer);
while (poses2.length < maxDetected && !queue.empty()) {
const root = queue.dequeue();
const rootImageCoords = getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
const rootImageCoords = getImageCoords(root.part, outputStride, offsetsBuffer);
if (withinRadius(poses2, rootImageCoords, root.part.id))
continue;
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
const keypoints = allKeypoints.filter((a) => a.score > minConfidence);
let keypoints = decodePose(root, scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer);
keypoints = keypoints.filter((a) => a.score > minConfidence);
const score = getInstanceScore(poses2, keypoints);
const box4 = getBoundingBox(keypoints);
if (score > minConfidence)
@ -18206,105 +18205,6 @@ __export(draw_exports, {
object: () => object,
options: () => options
});
// src/config.ts
var config2 = {
backend: "webgl",
modelBasePath: "../models/",
wasmPath: "../assets/",
debug: true,
async: true,
videoOptimized: true,
warmup: "full",
filter: {
enabled: true,
width: 0,
height: 0,
flip: false,
return: true,
brightness: 0,
contrast: 0,
sharpness: 0,
blur: 0,
saturation: 0,
hue: 0,
negative: false,
sepia: false,
vintage: false,
kodachrome: false,
technicolor: false,
polaroid: false,
pixelate: 0
},
gesture: {
enabled: true
},
face: {
enabled: true,
detector: {
modelPath: "blazeface.json",
rotation: false,
maxDetected: 10,
skipFrames: 21,
skipInitial: false,
minConfidence: 0.2,
iouThreshold: 0.1,
return: false
},
mesh: {
enabled: true,
modelPath: "facemesh.json"
},
iris: {
enabled: true,
modelPath: "iris.json"
},
description: {
enabled: true,
modelPath: "faceres.json",
skipFrames: 31,
minConfidence: 0.1
},
emotion: {
enabled: true,
minConfidence: 0.1,
skipFrames: 32,
modelPath: "emotion.json"
}
},
body: {
enabled: true,
modelPath: "posenet.json",
maxDetected: 1,
minConfidence: 0.2
},
hand: {
enabled: true,
rotation: false,
skipFrames: 12,
skipInitial: false,
minConfidence: 0.1,
iouThreshold: 0.1,
maxDetected: 1,
landmarks: true,
detector: {
modelPath: "handdetect.json"
},
skeleton: {
modelPath: "handskeleton.json"
}
},
object: {
enabled: false,
modelPath: "nanodet.json",
minConfidence: 0.2,
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 41
}
};
// src/draw/draw.ts
var options = {
color: "rgba(173, 216, 230, 0.3)",
labelColor: "rgba(173, 216, 230, 1)",
@ -18562,87 +18462,87 @@ async function body2(inCanvas2, result, drawOptions) {
const points = [];
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
if (points.length === 4)
lines(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftKnee");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftAnkle");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftHeel");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftFoot");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightKnee");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightAnkle");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightHeel");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightFoot");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftElbow");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftWrist");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftPalm");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightElbow");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightWrist");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightPalm");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
}
@ -18792,6 +18692,103 @@ async function all(inCanvas2, result, drawOptions) {
object(inCanvas2, result.object, localOptions);
}
// src/config.ts
var config2 = {
backend: "webgl",
modelBasePath: "../models/",
wasmPath: "../assets/",
debug: true,
async: true,
videoOptimized: true,
warmup: "full",
filter: {
enabled: true,
width: 0,
height: 0,
flip: false,
return: true,
brightness: 0,
contrast: 0,
sharpness: 0,
blur: 0,
saturation: 0,
hue: 0,
negative: false,
sepia: false,
vintage: false,
kodachrome: false,
technicolor: false,
polaroid: false,
pixelate: 0
},
gesture: {
enabled: true
},
face: {
enabled: true,
detector: {
modelPath: "blazeface.json",
rotation: false,
maxDetected: 10,
skipFrames: 21,
skipInitial: false,
minConfidence: 0.2,
iouThreshold: 0.1,
return: false
},
mesh: {
enabled: true,
modelPath: "facemesh.json"
},
iris: {
enabled: true,
modelPath: "iris.json"
},
description: {
enabled: true,
modelPath: "faceres.json",
skipFrames: 31,
minConfidence: 0.1
},
emotion: {
enabled: true,
minConfidence: 0.1,
skipFrames: 32,
modelPath: "emotion.json"
}
},
body: {
enabled: true,
modelPath: "posenet.json",
maxDetected: 1,
minConfidence: 0.1
},
hand: {
enabled: true,
rotation: false,
skipFrames: 12,
skipInitial: false,
minConfidence: 0.1,
iouThreshold: 0.1,
maxDetected: 1,
landmarks: true,
detector: {
modelPath: "handdetect.json"
},
skeleton: {
modelPath: "handskeleton.json"
}
},
object: {
enabled: false,
modelPath: "nanodet.json",
minConfidence: 0.2,
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 41
}
};
// src/sample.ts
var face3 = `
/9j/4AAQSkZJRgABAQEAYABgAAD/4QBoRXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUA

277
dist/human.node.js vendored
View File

@ -4533,12 +4533,12 @@ function getOffsetPoint(y, x, keypoint, offsets) {
x: offsets.get(y, x, keypoint + count)
};
}
function getImageCoords(part, outputStride, offsets) {
function getImageCoords(part, outputStride2, offsets) {
const {heatmapY, heatmapX, id: keypoint} = part;
const {y, x} = getOffsetPoint(heatmapY, heatmapX, keypoint, offsets);
return {
x: part.heatmapX * outputStride + x,
y: part.heatmapY * outputStride + y
x: part.heatmapX * outputStride2 + x,
y: part.heatmapY * outputStride2 + y
};
}
function clamp(a, min, max) {
@ -4559,9 +4559,9 @@ function addVectors(a, b) {
// src/posenet/poses.ts
var localMaximumRadius = 1;
var defaultOutputStride = 16;
var squaredNmsRadius = 20 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
var outputStride = 16;
var squaredNmsRadius = 50 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, displacements, offsetRefineStep = 2) {
const getDisplacement = (point2) => ({
y: displacements.get(point2.y, point2.x, edgeId),
x: displacements.get(point2.y, point2.x, displacements.shape[2] / 2 + edgeId)
@ -4587,32 +4587,31 @@ function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scor
const score = scoresBuffer.get(targetKeyPointIndices.y, targetKeyPointIndices.x, targetKeypointId);
return {position: targetKeypoint, part: partNames[targetKeypointId], score};
}
function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) {
function decodePose(root, scores, offsets, displacementsFwd, displacementsBwd) {
const parentChildrenTuples = poseChain.map(([parentJoinName, childJoinName]) => [partIds[parentJoinName], partIds[childJoinName]]);
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
const numParts = scores.shape[2];
const numEdges = parentToChildEdges.length;
const instanceKeypoints = new Array(numParts);
const {part: rootPart, score: rootScore} = root;
const rootPoint = getImageCoords(rootPart, outputStride, offsets);
instanceKeypoints[rootPart.id] = {
score: rootScore,
part: partNames[rootPart.id],
const rootPoint = getImageCoords(root.part, outputStride, offsets);
instanceKeypoints[root.part.id] = {
score: root.score,
part: partNames[root.part.id],
position: rootPoint
};
for (let edge = numEdges - 1; edge >= 0; --edge) {
const sourceKeypointId = parentToChildEdges[edge];
const targetKeypointId = childToParentEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsBwd);
}
}
for (let edge = 0; edge < numEdges; ++edge) {
const sourceKeypointId = childToParentEdges[edge];
const targetKeypointId = parentToChildEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsFwd);
}
}
return instanceKeypoints;
@ -4671,11 +4670,11 @@ function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacemen
const queue = buildPartWithScoreQueue(minConfidence, scoresBuffer);
while (poses2.length < maxDetected && !queue.empty()) {
const root = queue.dequeue();
const rootImageCoords = getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
const rootImageCoords = getImageCoords(root.part, outputStride, offsetsBuffer);
if (withinRadius(poses2, rootImageCoords, root.part.id))
continue;
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
const keypoints = allKeypoints.filter((a) => a.score > minConfidence);
let keypoints = decodePose(root, scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer);
keypoints = keypoints.filter((a) => a.score > minConfidence);
const score = getInstanceScore(poses2, keypoints);
const box4 = getBoundingBox(keypoints);
if (score > minConfidence)
@ -18205,105 +18204,6 @@ __export(draw_exports, {
object: () => object,
options: () => options
});
// src/config.ts
var config2 = {
backend: "webgl",
modelBasePath: "../models/",
wasmPath: "../assets/",
debug: true,
async: true,
videoOptimized: true,
warmup: "full",
filter: {
enabled: true,
width: 0,
height: 0,
flip: false,
return: true,
brightness: 0,
contrast: 0,
sharpness: 0,
blur: 0,
saturation: 0,
hue: 0,
negative: false,
sepia: false,
vintage: false,
kodachrome: false,
technicolor: false,
polaroid: false,
pixelate: 0
},
gesture: {
enabled: true
},
face: {
enabled: true,
detector: {
modelPath: "blazeface.json",
rotation: false,
maxDetected: 10,
skipFrames: 21,
skipInitial: false,
minConfidence: 0.2,
iouThreshold: 0.1,
return: false
},
mesh: {
enabled: true,
modelPath: "facemesh.json"
},
iris: {
enabled: true,
modelPath: "iris.json"
},
description: {
enabled: true,
modelPath: "faceres.json",
skipFrames: 31,
minConfidence: 0.1
},
emotion: {
enabled: true,
minConfidence: 0.1,
skipFrames: 32,
modelPath: "emotion.json"
}
},
body: {
enabled: true,
modelPath: "posenet.json",
maxDetected: 1,
minConfidence: 0.2
},
hand: {
enabled: true,
rotation: false,
skipFrames: 12,
skipInitial: false,
minConfidence: 0.1,
iouThreshold: 0.1,
maxDetected: 1,
landmarks: true,
detector: {
modelPath: "handdetect.json"
},
skeleton: {
modelPath: "handskeleton.json"
}
},
object: {
enabled: false,
modelPath: "nanodet.json",
minConfidence: 0.2,
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 41
}
};
// src/draw/draw.ts
var options = {
color: "rgba(173, 216, 230, 0.3)",
labelColor: "rgba(173, 216, 230, 1)",
@ -18561,87 +18461,87 @@ async function body2(inCanvas2, result, drawOptions) {
const points = [];
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
if (points.length === 4)
lines(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftKnee");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftAnkle");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftHeel");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftFoot");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightHip");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightKnee");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightAnkle");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightHeel");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightFoot");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "leftShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftElbow");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftWrist");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "leftPalm");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
points.length = 0;
part = result[i].keypoints.find((a) => a.part === "rightShoulder");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightElbow");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightWrist");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === "rightPalm");
if (part && part.score > config2.body.minConfidence)
if (part)
points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
}
@ -18791,6 +18691,103 @@ async function all(inCanvas2, result, drawOptions) {
object(inCanvas2, result.object, localOptions);
}
// src/config.ts
var config2 = {
backend: "webgl",
modelBasePath: "../models/",
wasmPath: "../assets/",
debug: true,
async: true,
videoOptimized: true,
warmup: "full",
filter: {
enabled: true,
width: 0,
height: 0,
flip: false,
return: true,
brightness: 0,
contrast: 0,
sharpness: 0,
blur: 0,
saturation: 0,
hue: 0,
negative: false,
sepia: false,
vintage: false,
kodachrome: false,
technicolor: false,
polaroid: false,
pixelate: 0
},
gesture: {
enabled: true
},
face: {
enabled: true,
detector: {
modelPath: "blazeface.json",
rotation: false,
maxDetected: 10,
skipFrames: 21,
skipInitial: false,
minConfidence: 0.2,
iouThreshold: 0.1,
return: false
},
mesh: {
enabled: true,
modelPath: "facemesh.json"
},
iris: {
enabled: true,
modelPath: "iris.json"
},
description: {
enabled: true,
modelPath: "faceres.json",
skipFrames: 31,
minConfidence: 0.1
},
emotion: {
enabled: true,
minConfidence: 0.1,
skipFrames: 32,
modelPath: "emotion.json"
}
},
body: {
enabled: true,
modelPath: "posenet.json",
maxDetected: 1,
minConfidence: 0.1
},
hand: {
enabled: true,
rotation: false,
skipFrames: 12,
skipInitial: false,
minConfidence: 0.1,
iouThreshold: 0.1,
maxDetected: 1,
landmarks: true,
detector: {
modelPath: "handdetect.json"
},
skeleton: {
modelPath: "handskeleton.json"
}
},
object: {
enabled: false,
modelPath: "nanodet.json",
minConfidence: 0.2,
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 41
}
};
// src/sample.ts
var face3 = `
/9j/4AAQSkZJRgABAQEAYABgAAD/4QBoRXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUA

BIN
models/posenet.bin
View File

Binary file not shown.

266
models/posenet.json
View File

File diff suppressed because one or more lines are too long

17
server/build.log
View File

@ -474,3 +474,20 @@
2021-05-04 11:17:56 INFO:  Generate types: ["src/human.ts"]
2021-05-04 11:18:01 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-05-04 11:18:01 INFO:  Generate TypeDocs: ["src/human.ts"]
2021-05-04 20:45:53 INFO:  @vladmandic/human version 1.8.2
2021-05-04 20:45:53 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-05-04 20:45:53 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-05-04 20:45:53 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":696,"outputFiles":"dist/tfjs.esm.js"}
2021-05-04 20:45:53 STATE: Build for: node type: node: {"imports":35,"importBytes":413741,"outputBytes":372378,"outputFiles":"dist/human.node.js"}
2021-05-04 20:45:53 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":700,"outputFiles":"dist/tfjs.esm.js"}
2021-05-04 20:45:53 STATE: Build for: nodeGPU type: node: {"imports":35,"importBytes":413745,"outputBytes":372382,"outputFiles":"dist/human.node-gpu.js"}
2021-05-04 20:45:53 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":746,"outputFiles":"dist/tfjs.esm.js"}
2021-05-04 20:45:53 STATE: Build for: nodeWASM type: node: {"imports":35,"importBytes":413791,"outputBytes":372432,"outputFiles":"dist/human.node-wasm.js"}
2021-05-04 20:45:53 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-05-04 20:45:53 STATE: Build for: browserNoBundle type: esm: {"imports":35,"importBytes":414439,"outputBytes":229511,"outputFiles":"dist/human.esm-nobundle.js"}
2021-05-04 20:45:54 STATE: Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2488,"outputBytes":1111306,"outputFiles":"dist/tfjs.esm.js"}
2021-05-04 20:45:55 STATE: Build for: browserBundle type: iife: {"imports":35,"importBytes":1524351,"outputBytes":1337203,"outputFiles":"dist/human.js"}
2021-05-04 20:45:55 STATE: Build for: browserBundle type: esm: {"imports":35,"importBytes":1524351,"outputBytes":1337161,"outputFiles":"dist/human.esm.js"}
2021-05-04 20:45:55 INFO:  Generate types: ["src/human.ts"]
2021-05-04 20:46:01 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-05-04 20:46:01 INFO:  Generate TypeDocs: ["src/human.ts"]

2
src/config.ts
View File

@ -302,7 +302,7 @@ const config: Config = {
maxDetected: 1, // maximum number of people detected in the input
// should be set to the minimum number for performance
// only valid for posenet as blazepose only detects single pose
minConfidence: 0.2, // threshold for discarding a prediction
minConfidence: 0.1, // threshold for discarding a prediction
},
hand: {

49
src/draw/draw.ts
View File

@ -1,4 +1,3 @@
import { defaults } from '../config';
import { TRI468 as triangulation } from '../blazeface/coords';
import { mergeDeep } from '../helpers';
@ -294,68 +293,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, draw
// shoulder line
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// torso main
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
// leg left
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// leg right
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// arm left
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// arm right
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// draw all
}

1
src/posenet/posenet.ts
View File

@ -21,7 +21,6 @@ export async function predict(input, config) {
const decoded = await poses.decode(buffers[0], buffers[1], buffers[2], buffers[3], config.body.maxDetected, config.body.minConfidence);
const scaled = util.scalePoses(decoded, [input.shape[1], input.shape[2]], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
return scaled;
}

30
src/posenet/poses.ts
View File

@ -2,10 +2,10 @@ import * as utils from './utils';
import * as kpt from './keypoints';
const localMaximumRadius = 1;
const defaultOutputStride = 16;
const squaredNmsRadius = 20 ** 2;
const outputStride = 16;
const squaredNmsRadius = 50 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, displacements, offsetRefineStep = 2) {
const getDisplacement = (point) => ({
y: displacements.get(point.y, point.x, edgeId),
x: displacements.get(point.y, point.x, (displacements.shape[2] / 2) + edgeId),
@ -34,19 +34,19 @@ function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scor
return { position: targetKeypoint, part: kpt.partNames[targetKeypointId], score };
}
export function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) {
export function decodePose(root, scores, offsets, displacementsFwd, displacementsBwd) {
const parentChildrenTuples = kpt.poseChain.map(([parentJoinName, childJoinName]) => ([kpt.partIds[parentJoinName], kpt.partIds[childJoinName]]));
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
const numParts = scores.shape[2];
const numParts = scores.shape[2]; // [21,21,17]
const numEdges = parentToChildEdges.length;
const instanceKeypoints = new Array(numParts);
// Start a new detection instance at the position of the root.
const { part: rootPart, score: rootScore } = root;
const rootPoint = utils.getImageCoords(rootPart, outputStride, offsets);
instanceKeypoints[rootPart.id] = {
score: rootScore,
part: kpt.partNames[rootPart.id],
// const { part: rootPart, score: rootScore } = root;
const rootPoint = utils.getImageCoords(root.part, outputStride, offsets);
instanceKeypoints[root.part.id] = {
score: root.score,
part: kpt.partNames[root.part.id],
position: rootPoint,
};
// Decode the part positions upwards in the tree, following the backward displacements.
@ -54,7 +54,7 @@ export function decodePose(root, scores, offsets, outputStride, displacementsFwd
const sourceKeypointId = parentToChildEdges[edge];
const targetKeypointId = childToParentEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsBwd);
}
}
// Decode the part positions downwards in the tree, following the forward displacements.
@ -62,7 +62,7 @@ export function decodePose(root, scores, offsets, outputStride, displacementsFwd
const sourceKeypointId = childToParentEdges[edge];
const targetKeypointId = parentToChildEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsFwd);
}
}
return instanceKeypoints;
@ -127,11 +127,11 @@ export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, disp
// The top element in the queue is the next root candidate.
const root = queue.dequeue();
// Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
const rootImageCoords = utils.getImageCoords(root.part, outputStride, offsetsBuffer);
if (withinRadius(poses, rootImageCoords, root.part.id)) continue;
// Else start a new detection instance at the position of the root.
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
const keypoints = allKeypoints.filter((a) => a.score > minConfidence);
let keypoints = decodePose(root, scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer);
keypoints = keypoints.filter((a) => a.score > minConfidence);
const score = getInstanceScore(poses, keypoints);
const box = utils.getBoundingBox(keypoints);
if (score > minConfidence) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });

2
types/posenet/poses.d.ts vendored
View File

@ -1,5 +1,5 @@
import * as utils from './utils';
export declare function decodePose(root: any, scores: any, offsets: any, outputStride: any, displacementsFwd: any, displacementsBwd: any): any[];
export declare function decodePose(root: any, scores: any, offsets: any, displacementsFwd: any, displacementsBwd: any): any[];
export declare function buildPartWithScoreQueue(minConfidence: any, scores: any): utils.MaxHeap;
export declare function decode(offsetsBuffer: any, scoresBuffer: any, displacementsFwdBuffer: any, displacementsBwdBuffer: any, maxDetected: any, minConfidence: any): {
keypoints: any;