ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switch posenet weights

pull/280/head
Vladimir Mandic 2021-05-04 20:46:33 -04:00
parent 31a794a8db
commit 0120290047
5 changed files with 51 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -9,6 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog
### **HEAD -> main** 2021/05/04 mandic00@live.com
### **1.8.2** 2021/05/04 mandic00@live.com

16
demo/index.js
View File

@ -9,17 +9,17 @@ import webRTC from './helpers/webrtc.js';
let human;
const userConfig = {
warmup: 'none',
warmup: 'full',
/*
backend: 'humangl',
async: false,
profile: false,
videoOptimized: true,
backend: 'webgl',
async: true,
videoOptimized: false,
filter: {
enabled: false,
flip: false,
},
face: { enabled: true,
face: { enabled: false,
detector: { return: false },
mesh: { enabled: true },
iris: { enabled: true },
@ -28,7 +28,7 @@ const userConfig = {
},
hand: { enabled: false },
gesture: { enabled: false },
body: { enabled: false, modelPath: 'posenet.json' },
body: { enabled: true, modelPath: 'posenet.json' },
// body: { enabled: true, modelPath: 'blazepose.json' },
// object: { enabled: true },
*/
@ -58,7 +58,7 @@ const ui = {
detectFPS: [], // internal, holds fps values for detection performance
drawFPS: [], // internal, holds fps values for draw performance
buffered: true, // should output be buffered between frames
drawWarmup: false, // debug only, should warmup image processing be displayed on startup
drawWarmup: true, // debug only, should warmup image processing be displayed on startup
drawThread: null, // internl, perform draw operations in a separate thread
detectThread: null, // internl, perform detect operations in a separate thread
framesDraw: 0, // internal, statistics on frames drawn

2
src/config.ts
View File

@ -302,7 +302,7 @@ const config: Config = {
maxDetected: 1, // maximum number of people detected in the input
// should be set to the minimum number for performance
// only valid for posenet as blazepose only detects single pose
minConfidence: 0.2, // threshold for discarding a prediction
minConfidence: 0.1, // threshold for discarding a prediction
},
hand: {

49
src/draw/draw.ts
View File

@ -1,4 +1,3 @@
import { defaults } from '../config';
import { TRI468 as triangulation } from '../blazeface/coords';
import { mergeDeep } from '../helpers';
@ -294,68 +293,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, draw
// shoulder line
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// torso main
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
// leg left
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// leg right
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// arm left
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// arm right
points.length = 0;
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]);
if (part) points.push([part.position.x, part.position.y]);
curves(ctx, points, localOptions);
// draw all
}

30
src/posenet/poses.ts
View File

@ -2,10 +2,10 @@ import * as utils from './utils';
import * as kpt from './keypoints';
const localMaximumRadius = 1;
const defaultOutputStride = 16;
const squaredNmsRadius = 20 ** 2;
const outputStride = 16;
const squaredNmsRadius = 50 ** 2;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, displacements, offsetRefineStep = 2) {
const getDisplacement = (point) => ({
y: displacements.get(point.y, point.x, edgeId),
x: displacements.get(point.y, point.x, (displacements.shape[2] / 2) + edgeId),
@ -34,19 +34,19 @@ function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scor
return { position: targetKeypoint, part: kpt.partNames[targetKeypointId], score };
}
export function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) {
export function decodePose(root, scores, offsets, displacementsFwd, displacementsBwd) {
const parentChildrenTuples = kpt.poseChain.map(([parentJoinName, childJoinName]) => ([kpt.partIds[parentJoinName], kpt.partIds[childJoinName]]));
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
const numParts = scores.shape[2];
const numParts = scores.shape[2]; // [21,21,17]
const numEdges = parentToChildEdges.length;
const instanceKeypoints = new Array(numParts);
// Start a new detection instance at the position of the root.
const { part: rootPart, score: rootScore } = root;
const rootPoint = utils.getImageCoords(rootPart, outputStride, offsets);
instanceKeypoints[rootPart.id] = {
score: rootScore,
part: kpt.partNames[rootPart.id],
// const { part: rootPart, score: rootScore } = root;
const rootPoint = utils.getImageCoords(root.part, outputStride, offsets);
instanceKeypoints[root.part.id] = {
score: root.score,
part: kpt.partNames[root.part.id],
position: rootPoint,
};
// Decode the part positions upwards in the tree, following the backward displacements.
@ -54,7 +54,7 @@ export function decodePose(root, scores, offsets, outputStride, displacementsFwd
const sourceKeypointId = parentToChildEdges[edge];
const targetKeypointId = childToParentEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsBwd);
}
}
// Decode the part positions downwards in the tree, following the forward displacements.
@ -62,7 +62,7 @@ export function decodePose(root, scores, offsets, outputStride, displacementsFwd
const sourceKeypointId = childToParentEdges[edge];
const targetKeypointId = parentToChildEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd);
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsFwd);
}
}
return instanceKeypoints;
@ -127,11 +127,11 @@ export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, disp
// The top element in the queue is the next root candidate.
const root = queue.dequeue();
// Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
const rootImageCoords = utils.getImageCoords(root.part, outputStride, offsetsBuffer);
if (withinRadius(poses, rootImageCoords, root.part.id)) continue;
// Else start a new detection instance at the position of the root.
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
const keypoints = allKeypoints.filter((a) => a.score > minConfidence);
let keypoints = decodePose(root, scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer);
keypoints = keypoints.filter((a) => a.score > minConfidence);
const score = getInstanceScore(poses, keypoints);
const box = utils.getBoundingBox(keypoints);
if (score > minConfidence) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });