ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switch posenet weights

pull/280/head
Vladimir Mandic 2021-05-04 20:46:33 -04:00
parent 31a794a8db
commit 0120290047
5 changed files with 51 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -9,6 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog ## Changelog
### **HEAD -> main** 2021/05/04 mandic00@live.com
### **1.8.2** 2021/05/04 mandic00@live.com ### **1.8.2** 2021/05/04 mandic00@live.com

16
demo/index.js
View File

@ -9,17 +9,17 @@ import webRTC from './helpers/webrtc.js';
let human; let human;
const userConfig = { const userConfig = {
warmup: 'none', warmup: 'full',
/* /*
backend: 'humangl', backend: 'webgl',
async: false, async: true,
profile: false,
videoOptimized: true, videoOptimized: false,
filter: { filter: {
enabled: false, enabled: false,
flip: false, flip: false,
}, },
face: { enabled: true, face: { enabled: false,
detector: { return: false }, detector: { return: false },
mesh: { enabled: true }, mesh: { enabled: true },
iris: { enabled: true }, iris: { enabled: true },
@ -28,7 +28,7 @@ const userConfig = {
}, },
hand: { enabled: false }, hand: { enabled: false },
gesture: { enabled: false }, gesture: { enabled: false },
body: { enabled: false, modelPath: 'posenet.json' }, body: { enabled: true, modelPath: 'posenet.json' },
// body: { enabled: true, modelPath: 'blazepose.json' }, // body: { enabled: true, modelPath: 'blazepose.json' },
// object: { enabled: true }, // object: { enabled: true },
*/ */
@ -58,7 +58,7 @@ const ui = {
detectFPS: [], // internal, holds fps values for detection performance detectFPS: [], // internal, holds fps values for detection performance
drawFPS: [], // internal, holds fps values for draw performance drawFPS: [], // internal, holds fps values for draw performance
buffered: true, // should output be buffered between frames buffered: true, // should output be buffered between frames
drawWarmup: false, // debug only, should warmup image processing be displayed on startup drawWarmup: true, // debug only, should warmup image processing be displayed on startup
drawThread: null, // internl, perform draw operations in a separate thread drawThread: null, // internl, perform draw operations in a separate thread
detectThread: null, // internl, perform detect operations in a separate thread detectThread: null, // internl, perform detect operations in a separate thread
framesDraw: 0, // internal, statistics on frames drawn framesDraw: 0, // internal, statistics on frames drawn

2
src/config.ts
View File

@ -302,7 +302,7 @@ const config: Config = {
maxDetected: 1, // maximum number of people detected in the input maxDetected: 1, // maximum number of people detected in the input
// should be set to the minimum number for performance // should be set to the minimum number for performance
// only valid for posenet as blazepose only detects single pose // only valid for posenet as blazepose only detects single pose
minConfidence: 0.2, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
}, },
hand: { hand: {

49
src/draw/draw.ts
View File

@ -1,4 +1,3 @@
import { defaults } from '../config';
import { TRI468 as triangulation } from '../blazeface/coords'; import { TRI468 as triangulation } from '../blazeface/coords';
import { mergeDeep } from '../helpers'; import { mergeDeep } from '../helpers';
@ -294,68 +293,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, draw
// shoulder line // shoulder line
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// torso main // torso main
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHip'); part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHip'); part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
// leg left // leg left
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftHip'); part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftKnee'); part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHeel'); part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftFoot'); part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// leg right // leg right
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightHip'); part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightKnee'); part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHeel'); part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightFoot'); part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// arm left // arm left
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftElbow'); part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftWrist'); part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftPalm'); part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// arm right // arm right
points.length = 0; points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightElbow'); part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightWrist'); part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightPalm'); part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions); curves(ctx, points, localOptions);
// draw all // draw all
} }

30
src/posenet/poses.ts
View File

@ -2,10 +2,10 @@ import * as utils from './utils';
import * as kpt from './keypoints'; import * as kpt from './keypoints';
const localMaximumRadius = 1; const localMaximumRadius = 1;
const defaultOutputStride = 16; const outputStride = 16;
const squaredNmsRadius = 20 ** 2; const squaredNmsRadius = 50 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) { function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, displacements, offsetRefineStep = 2) {
const getDisplacement = (point) => ({ const getDisplacement = (point) => ({
y: displacements.get(point.y, point.x, edgeId), y: displacements.get(point.y, point.x, edgeId),
x: displacements.get(point.y, point.x, (displacements.shape[2] / 2) + edgeId), x: displacements.get(point.y, point.x, (displacements.shape[2] / 2) + edgeId),
@ -34,19 +34,19 @@ function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scor
return { position: targetKeypoint, part: kpt.partNames[targetKeypointId], score }; return { position: targetKeypoint, part: kpt.partNames[targetKeypointId], score };
} }
export function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) { export function decodePose(root, scores, offsets, displacementsFwd, displacementsBwd) {
const parentChildrenTuples = kpt.poseChain.map(([parentJoinName, childJoinName]) => ([kpt.partIds[parentJoinName], kpt.partIds[childJoinName]])); const parentChildrenTuples = kpt.poseChain.map(([parentJoinName, childJoinName]) => ([kpt.partIds[parentJoinName], kpt.partIds[childJoinName]]));
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId); const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId); const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
const numParts = scores.shape[2]; const numParts = scores.shape[2]; // [21,21,17]
const numEdges = parentToChildEdges.length; const numEdges = parentToChildEdges.length;
const instanceKeypoints = new Array(numParts); const instanceKeypoints = new Array(numParts);
// Start a new detection instance at the position of the root. // Start a new detection instance at the position of the root.
const { part: rootPart, score: rootScore } = root; // const { part: rootPart, score: rootScore } = root;
const rootPoint = utils.getImageCoords(rootPart, outputStride, offsets); const rootPoint = utils.getImageCoords(root.part, outputStride, offsets);
instanceKeypoints[rootPart.id] = { instanceKeypoints[root.part.id] = {
score: rootScore, score: root.score,
part: kpt.partNames[rootPart.id], part: kpt.partNames[root.part.id],
position: rootPoint, position: rootPoint,
}; };
// Decode the part positions upwards in the tree, following the backward displacements. // Decode the part positions upwards in the tree, following the backward displacements.
@ -54,7 +54,7 @@ export function decodePose(root, scores, offsets, outputStride, displacementsFwd
const sourceKeypointId = parentToChildEdges[edge]; const sourceKeypointId = parentToChildEdges[edge];
const targetKeypointId = childToParentEdges[edge]; const targetKeypointId = childToParentEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) { if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd); instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsBwd);
} }
} }
// Decode the part positions downwards in the tree, following the forward displacements. // Decode the part positions downwards in the tree, following the forward displacements.
@ -62,7 +62,7 @@ export function decodePose(root, scores, offsets, outputStride, displacementsFwd
const sourceKeypointId = childToParentEdges[edge]; const sourceKeypointId = childToParentEdges[edge];
const targetKeypointId = parentToChildEdges[edge]; const targetKeypointId = parentToChildEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) { if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd); instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsFwd);
} }
} }
return instanceKeypoints; return instanceKeypoints;
@ -127,11 +127,11 @@ export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, disp
// The top element in the queue is the next root candidate. // The top element in the queue is the next root candidate.
const root = queue.dequeue(); const root = queue.dequeue();
// Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance. // Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer); const rootImageCoords = utils.getImageCoords(root.part, outputStride, offsetsBuffer);
if (withinRadius(poses, rootImageCoords, root.part.id)) continue; if (withinRadius(poses, rootImageCoords, root.part.id)) continue;
// Else start a new detection instance at the position of the root. // Else start a new detection instance at the position of the root.
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer); let keypoints = decodePose(root, scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer);
const keypoints = allKeypoints.filter((a) => a.score > minConfidence); keypoints = keypoints.filter((a) => a.score > minConfidence);
const score = getInstanceScore(poses, keypoints); const score = getInstanceScore(poses, keypoints);
const box = utils.getBoundingBox(keypoints); const box = utils.getBoundingBox(keypoints);
if (score > minConfidence) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 }); if (score > minConfidence) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });