From a95ca54bbf8afef11d5f5a9b73f032e76c696406 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 May 2021 20:46:33 -0400 Subject: [PATCH] switch posenet weights --- CHANGELOG.md | 3 +++ demo/index.js | 16 +++++++-------- src/config.ts | 2 +- src/draw/draw.ts | 49 ++++++++++++++++++++++---------------------- src/posenet/poses.ts | 30 +++++++++++++-------------- 5 files changed, 51 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d65c100a..9eba2565 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ Repository: **** ## Changelog +### **HEAD -> main** 2021/05/04 mandic00@live.com + + ### **1.8.2** 2021/05/04 mandic00@live.com diff --git a/demo/index.js b/demo/index.js index be7cbcd7..341356e1 100644 --- a/demo/index.js +++ b/demo/index.js @@ -9,17 +9,17 @@ import webRTC from './helpers/webrtc.js'; let human; const userConfig = { - warmup: 'none', + warmup: 'full', /* - backend: 'humangl', - async: false, - profile: false, - videoOptimized: true, + backend: 'webgl', + async: true, + + videoOptimized: false, filter: { enabled: false, flip: false, }, - face: { enabled: true, + face: { enabled: false, detector: { return: false }, mesh: { enabled: true }, iris: { enabled: true }, @@ -28,7 +28,7 @@ const userConfig = { }, hand: { enabled: false }, gesture: { enabled: false }, - body: { enabled: false, modelPath: 'posenet.json' }, + body: { enabled: true, modelPath: 'posenet.json' }, // body: { enabled: true, modelPath: 'blazepose.json' }, // object: { enabled: true }, */ @@ -58,7 +58,7 @@ const ui = { detectFPS: [], // internal, holds fps values for detection performance drawFPS: [], // internal, holds fps values for draw performance buffered: true, // should output be buffered between frames - drawWarmup: false, // debug only, should warmup image processing be displayed on startup + drawWarmup: true, // debug only, should warmup image processing be displayed on startup drawThread: null, // internl, perform draw operations in a separate thread detectThread: null, // internl, perform detect operations in a separate thread framesDraw: 0, // internal, statistics on frames drawn diff --git a/src/config.ts b/src/config.ts index 662ec1f2..75f41cae 100644 --- a/src/config.ts +++ b/src/config.ts @@ -302,7 +302,7 @@ const config: Config = { maxDetected: 1, // maximum number of people detected in the input // should be set to the minimum number for performance // only valid for posenet as blazepose only detects single pose - minConfidence: 0.2, // threshold for discarding a prediction + minConfidence: 0.1, // threshold for discarding a prediction }, hand: { diff --git a/src/draw/draw.ts b/src/draw/draw.ts index e989e405..fdfe01c0 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -1,4 +1,3 @@ -import { defaults } from '../config'; import { TRI468 as triangulation } from '../blazeface/coords'; import { mergeDeep } from '../helpers'; @@ -294,68 +293,68 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, draw // shoulder line points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // torso main points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightHip'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftHip'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso // leg left points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftHip'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftKnee'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftHeel'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftFoot'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // leg right points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightHip'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightKnee'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightHeel'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightFoot'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // arm left points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftElbow'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftWrist'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftPalm'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // arm right points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightElbow'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightWrist'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightPalm'); - if (part && part.score > defaults.body.minConfidence) points.push([part.position.x, part.position.y]); + if (part) points.push([part.position.x, part.position.y]); curves(ctx, points, localOptions); // draw all } diff --git a/src/posenet/poses.ts b/src/posenet/poses.ts index 2d8cee59..3ffd471b 100644 --- a/src/posenet/poses.ts +++ b/src/posenet/poses.ts @@ -2,10 +2,10 @@ import * as utils from './utils'; import * as kpt from './keypoints'; const localMaximumRadius = 1; -const defaultOutputStride = 16; -const squaredNmsRadius = 20 ** 2; +const outputStride = 16; +const squaredNmsRadius = 50 ** 2; -function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) { +function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, displacements, offsetRefineStep = 2) { const getDisplacement = (point) => ({ y: displacements.get(point.y, point.x, edgeId), x: displacements.get(point.y, point.x, (displacements.shape[2] / 2) + edgeId), @@ -34,19 +34,19 @@ function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scor return { position: targetKeypoint, part: kpt.partNames[targetKeypointId], score }; } -export function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) { +export function decodePose(root, scores, offsets, displacementsFwd, displacementsBwd) { const parentChildrenTuples = kpt.poseChain.map(([parentJoinName, childJoinName]) => ([kpt.partIds[parentJoinName], kpt.partIds[childJoinName]])); const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId); const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId); - const numParts = scores.shape[2]; + const numParts = scores.shape[2]; // [21,21,17] const numEdges = parentToChildEdges.length; const instanceKeypoints = new Array(numParts); // Start a new detection instance at the position of the root. - const { part: rootPart, score: rootScore } = root; - const rootPoint = utils.getImageCoords(rootPart, outputStride, offsets); - instanceKeypoints[rootPart.id] = { - score: rootScore, - part: kpt.partNames[rootPart.id], + // const { part: rootPart, score: rootScore } = root; + const rootPoint = utils.getImageCoords(root.part, outputStride, offsets); + instanceKeypoints[root.part.id] = { + score: root.score, + part: kpt.partNames[root.part.id], position: rootPoint, }; // Decode the part positions upwards in the tree, following the backward displacements. @@ -54,7 +54,7 @@ export function decodePose(root, scores, offsets, outputStride, displacementsFwd const sourceKeypointId = parentToChildEdges[edge]; const targetKeypointId = childToParentEdges[edge]; if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) { - instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd); + instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsBwd); } } // Decode the part positions downwards in the tree, following the forward displacements. @@ -62,7 +62,7 @@ export function decodePose(root, scores, offsets, outputStride, displacementsFwd const sourceKeypointId = childToParentEdges[edge]; const targetKeypointId = parentToChildEdges[edge]; if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) { - instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd); + instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, displacementsFwd); } } return instanceKeypoints; @@ -127,11 +127,11 @@ export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, disp // The top element in the queue is the next root candidate. const root = queue.dequeue(); // Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance. - const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer); + const rootImageCoords = utils.getImageCoords(root.part, outputStride, offsetsBuffer); if (withinRadius(poses, rootImageCoords, root.part.id)) continue; // Else start a new detection instance at the position of the root. - const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer); - const keypoints = allKeypoints.filter((a) => a.score > minConfidence); + let keypoints = decodePose(root, scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer); + keypoints = keypoints.filter((a) => a.score > minConfidence); const score = getInstanceScore(poses, keypoints); const box = utils.getBoundingBox(keypoints); if (score > minConfidence) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });